-----============= acceptance-small: replay-dual ============----- Wed Aug 12 16:04:34 EDT 2026 MGS_OS_ID_LIKE=rhel centos fedora rocky MGS_OS_VERSION_ID=8.10 MGS_OS_ID=rocky MGS_OS_VERSION_CODE=134873088 MDS1_OS_VERSION_ID=8.10 MDS1_OS_VERSION_CODE=134873088 MDS1_OS_ID_LIKE=rhel centos fedora rocky MDS1_OS_ID=rocky OST1_OS_VERSION_CODE=134873088 OST1_OS_ID_LIKE=rhel centos fedora rocky OST1_OS_VERSION_ID=8.10 OST1_OS_ID=rocky CLIENT_OS_ID=rocky CLIENT_OS_VERSION_CODE=134873088 CLIENT_OS_VERSION_ID=8.10 CLIENT_OS_ID_LIKE=rhel centos fedora rocky oleg403-server: ls: cannot access '/home/green/git/lustre-release/lustre/tests/except/replay-dual.*ex': No such file or directory excepting tests: 14b 21b skipping tests SLOW=no: 21b === replay-dual: start setup 16:04:58 (1786565098) === Starting client oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 Mount client oleg403-client.virtnet: mount -t lustre -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 Started clients oleg403-client.virtnet: 192.168.204.103@tcp:/lustre on /mnt/lustre2 type lustre (rw,flock,checksum,encrypt,lazystatfs,lruresize,nolock,statfs_project,nosync_on_close,nouser_fid2path,user_xattr,verbose) oleg403-client.virtnet: executing check_config_client /mnt/lustre oleg403-client.virtnet: Checking config lustre mounted on /mnt/lustre Checking servers environments Checking clients oleg403-client.virtnet environments Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff93aa8948c000.idle_timeout=debug osc.lustre-OST0000-osc-ffff93aa911de000.idle_timeout=debug osc.lustre-OST0001-osc-ffff93aa8948c000.idle_timeout=debug osc.lustre-OST0001-osc-ffff93aa911de000.idle_timeout=debug disable quota as required oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all osd-ldiskfs.track_declares_assert=1 === replay-dual: finish setup 16:05:44 (1786565144) === == replay-dual test 0a: expired recovery with lost client ========================================================== 16:05:47 (1786565147) Check file is LU482_FAILED=/tmp/replay-dual.lu482.yGDDn3 UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1708 1269596 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1532 1269772 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre total: 50 open/close in 2.36 seconds: 21.22 ops/second fail_loc=0x80000514 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:06:04 (1786565164) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:06:40 (1786565200) targets are mounted 16:06:40 (1786565200) facet_failover done Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 - unlinked 0 (time 1786565313 ; total 0 ; last 0) total: 50 unlinks in 3 seconds: 16.666666 unlinks/second PASS 0a (175s) == replay-dual test 0b: lost client during waiting for next transno ========================================================== 16:08:42 (1786565322) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1676 1269628 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1532 1269772 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:08:57 (1786565337) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:09:30 (1786565370) targets are mounted 16:09:30 (1786565370) facet_failover done Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 PASS 0b (225s) == replay-dual test 1: |X| simple create ================= 16:12:27 (1786565547) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1676 1269628 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1532 1269772 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre mcreate: cannot create `/mnt/lustre/fsa-oleg403-client.virtnet' with mode 0100644: File exists Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:12:42 (1786565562) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:13:13 (1786565593) targets are mounted 16:13:14 (1786565594) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 1 (64s) == replay-dual test 2: |X| mkdir adir ==================== 16:13:31 (1786565611) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1676 1269628 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1532 1269772 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:13:45 (1786565625) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:14:17 (1786565657) targets are mounted 16:14:17 (1786565657) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 2 (64s) == replay-dual test 3: |X| mkdir adir, mkdir adir/bdir === 16:14:35 (1786565675) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1676 1269628 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1532 1269772 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:14:50 (1786565690) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:15:25 (1786565725) targets are mounted 16:15:25 (1786565725) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 3 (67s) == replay-dual test 4: |X| mkdir adir (-EEXIST), mkdir adir/bdir ========================================================== 16:15:42 (1786565742) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1852 1269452 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1704 1269600 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre mkdir: cannot create directory '/mnt/lustre/adir': File exists Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:15:57 (1786565757) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:16:36 (1786565796) targets are mounted 16:16:36 (1786565796) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 4 (71s) == replay-dual test 5: open, unlink |X| close ============ 16:16:53 (1786565813) multiop /mnt/lustre2/a vo_tSc TMPPIPE=/tmp/multiop_open_wait_pipe.8258 UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:17:07 (1786565827) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:17:43 (1786565863) targets are mounted 16:17:43 (1786565863) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 5 (67s) == replay-dual test 6: open1, open2, unlink |X| close1 [fail mds1] close2 ========================================================== 16:18:00 (1786565880) multiop /mnt/lustre2/a vo_c TMPPIPE=/tmp/multiop_open_wait_pipe.8258 multiop /mnt/lustre/a vo_c TMPPIPE=/tmp/multiop_open_wait_pipe.8258 UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:18:12 (1786565892) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:18:46 (1786565926) targets are mounted 16:18:46 (1786565926) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 6 (63s) == replay-dual test 8: replay of resent request ========== 16:19:03 (1786565943) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre fail_loc=0x119 fail_loc=0 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:19:36 (1786565976) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:20:12 (1786566012) targets are mounted 16:20:12 (1786566012) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 8 (87s) == replay-dual test 9: resending a replayed create ======= 16:20:30 (1786566030) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre fail_loc=0x80000119 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:20:46 (1786566046) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:21:20 (1786566080) targets are mounted 16:21:20 (1786566080) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec fail_loc=0 PASS 9 (75s) == replay-dual test 10: resending a replayed unlink ====== 16:21:45 (1786566105) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre fail_loc=0x80000119 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:22:00 (1786566120) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:22:32 (1786566152) targets are mounted 16:22:32 (1786566152) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec fail_loc=0 PASS 10 (73s) == replay-dual test 11: both clients timeout during replay ========================================================== 16:22:58 (1786566178) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre fail_loc=0x0119 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:23:11 (1786566191) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:23:45 (1786566225) targets are mounted 16:23:45 (1786566225) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount FULL mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 6 sec fail_loc=0 PASS 11 (65s) == replay-dual test 12: open resend timeout ============== 16:24:03 (1786566243) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre multiop /mnt/lustre/f12.replay-dual vmo_c TMPPIPE=/tmp/multiop_open_wait_pipe.8258 fail_loc=0x80000302 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:24:17 (1786566257) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:24:47 (1786566287) targets are mounted 16:24:47 (1786566287) facet_failover done fail_loc=0 /mnt/lustre/f12.replay-dual /mnt/lustre/f12.replay-dual has type file OK PASS 12 (66s) == replay-dual test 13: close resend timeout ============= 16:25:09 (1786566309) multiop /mnt/lustre/f13.replay-dual vmo_c TMPPIPE=/tmp/multiop_open_wait_pipe.8258 UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre fail_loc=0x80000115 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:25:21 (1786566321) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:25:56 (1786566356) targets are mounted 16:25:56 (1786566356) facet_failover done fail_loc=0 /mnt/lustre/f13.replay-dual /mnt/lustre/f13.replay-dual has type file OK PASS 13 (67s) SKIP: replay-dual test_14b skipping ALWAYS excluded test 14b == replay-dual test 15a: timeout waiting for lost client during replay, 1 client completes ========================================================== 16:26:18 (1786566378) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre total: 25 open/close in 0.96 seconds: 26.07 ops/second total: 1 open/close in 0.04 seconds: 26.33 ops/second Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:26:31 (1786566391) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:27:04 (1786566424) targets are mounted 16:27:04 (1786566424) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec - unlinked 0 (time 1786566500 ; total 0 ; last 0) total: 25 unlinks in 1 seconds: 25.000000 unlinks/second Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 PASS 15a (128s) == replay-dual test 15c: remove multiple OST orphans ===== 16:28:26 (1786566506) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:30:20 (1786566620) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:30:50 (1786566650) targets are mounted 16:30:50 (1786566650) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 PASS 15c (223s) == replay-dual test 16: fail MDS during recovery (3571) == 16:32:09 (1786566729) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre total: 25 open/close in 0.75 seconds: 33.37 ops/second total: 1 open/close in 0.03 seconds: 38.09 ops/second Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:32:21 (1786566741) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:32:57 (1786566777) targets are mounted 16:32:57 (1786566777) facet_failover done Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:33:21 (1786566801) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:33:42 (1786566822) targets are mounted 16:33:42 (1786566822) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec - unlinked 0 (time 1786566895 ; total 0 ; last 0) total: 25 unlinks in 1 seconds: 25.000000 unlinks/second Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 PASS 16 (170s) == replay-dual test 17: fail OST during recovery (3571) == 16:34:59 (1786566899) total: 25 open/close in 0.80 seconds: 31.27 ops/second total: 1 open/close in 0.06 seconds: 17.15 ops/second UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1788 1269516 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3088 7210952 1% /mnt/lustre Failing ost1 on oleg403-server Stopping /mnt/lustre-ost1 (opts:) on oleg403-server 16:35:10 (1786566910) shut down facet: ost1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover ost1 to oleg403-server mount facets: ost1 Start ost1: mount -t lustre -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-OST0000 16:35:33 (1786566933) targets are mounted 16:35:33 (1786566933) facet_failover done Failing ost1 on oleg403-server Stopping /mnt/lustre-ost1 (opts:) on oleg403-server 16:35:57 (1786566957) shut down facet: ost1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover ost1 to oleg403-server mount facets: ost1 Start ost1: mount -t lustre -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-OST0000 16:36:33 (1786566993) targets are mounted 16:36:33 (1786566993) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid 1475 0 osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid in FULL state after 0 sec - unlinked 0 (time 1786567060 ; total 0 ; last 0) total: 25 unlinks in 1 seconds: 25.000000 unlinks/second Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 PASS 17 (169s) == replay-dual test 18: ldlm_handle_enqueue succeeds on evicted export (3822) ========================================================== 16:37:48 (1786567068) debug=+dlmtrace fail_loc=0x8000030b using seed 1517891523 running for 500 iterations total: 500 stats in 1 seconds: 500.000000 stats/second ldlm.namespaces.MGC192.168.204.103@tcp.early_lock_cancel=0 ldlm.namespaces.lustre-MDT0000-mdc-ffff93aa8732b000.early_lock_cancel=0 ldlm.namespaces.lustre-MDT0000-mdc-ffff93aa982e8800.early_lock_cancel=0 ldlm.namespaces.lustre-MDT0001-mdc-ffff93aa8732b000.early_lock_cancel=0 ldlm.namespaces.lustre-MDT0001-mdc-ffff93aa982e8800.early_lock_cancel=0 ldlm.namespaces.lustre-OST0000-osc-ffff93aa8732b000.early_lock_cancel=0 ldlm.namespaces.lustre-OST0000-osc-ffff93aa982e8800.early_lock_cancel=0 ldlm.namespaces.lustre-OST0001-osc-ffff93aa8732b000.early_lock_cancel=0 ldlm.namespaces.lustre-OST0001-osc-ffff93aa982e8800.early_lock_cancel=0 fail_loc=0x80000305 Error in opening file "/mnt/lustre2/d18.replay-dual/f18.replay-dual"(flags=O_RDONLY) 2: No such file or directory ldlm.namespaces.MGC192.168.204.103@tcp.early_lock_cancel=1 ldlm.namespaces.lustre-MDT0000-mdc-ffff93aa8732b000.early_lock_cancel=1 ldlm.namespaces.lustre-MDT0000-mdc-ffff93aa982e8800.early_lock_cancel=1 ldlm.namespaces.lustre-MDT0001-mdc-ffff93aa8732b000.early_lock_cancel=1 ldlm.namespaces.lustre-MDT0001-mdc-ffff93aa982e8800.early_lock_cancel=1 ldlm.namespaces.lustre-OST0000-osc-ffff93aa8732b000.early_lock_cancel=1 ldlm.namespaces.lustre-OST0000-osc-ffff93aa982e8800.early_lock_cancel=1 ldlm.namespaces.lustre-OST0001-osc-ffff93aa8732b000.early_lock_cancel=1 ldlm.namespaces.lustre-OST0001-osc-ffff93aa982e8800.early_lock_cancel=1 fail_loc=0 fail_loc=0 PASS 18 (62s) == replay-dual test 19: resend of open request =========== 16:38:50 (1786567130) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1792 1269512 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1560 3605460 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3104 7210936 1% /mnt/lustre fail_loc=0x157 - open/close 0 (time 1786567230.24 total 87.68 last 0.00) total: 1 open/close in 87.68 seconds: 0.01 ops/second fail_loc=0 Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:40:37 (1786567237) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:41:12 (1786567272) targets are mounted 16:41:12 (1786567272) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec PASS 19 (162s) == replay-dual test 20: recovery time is not increasing == 16:41:32 (1786567292) UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1792 1269512 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1560 3605460 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3104 7210936 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:41:50 (1786567310) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:42:27 (1786567347) targets are mounted 16:42:27 (1786567347) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec Starting client: oleg403-client.virtnet: -o user_xattr,flock 192.168.204.103@tcp:/lustre /mnt/lustre2 UUID 1K-blocks Used Available Use% Mounted on lustre-MDT0000_UUID 1414116 1792 1269512 1% /mnt/lustre[MDT:0] lustre-MDT0001_UUID 1414116 1668 1269636 1% /mnt/lustre[MDT:1] lustre-OST0000_UUID 3833116 1560 3605460 1% /mnt/lustre[OST:0] lustre-OST0001_UUID 3833116 1544 3605476 1% /mnt/lustre[OST:1] filesystem_summary: 7666232 3104 7210936 1% /mnt/lustre Failing mds1 on oleg403-server Stopping /mnt/lustre-mds1 (opts:) on oleg403-server 16:45:11 (1786567511) shut down facet: mds1 facet_host: oleg403-server facet_failover_host: oleg403-server Failover mds1 to oleg403-server mount facets: mds1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg403-server: oleg403-server.virtnet: executing set_default_debug -1 all pdsh@oleg403-client: oleg403-server: ssh exited with exit code 1 Started lustre-MDT0000 16:45:49 (1786567549) targets are mounted 16:45:49 (1786567549) facet_failover done oleg403-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec