== recovery-small test 51: failover MDS during recovery == 18:47:30 (1763336850)
fail_loc=0x00001310
Failing mds1 on oleg244-server
Stopping /mnt/lustre-mds1 (opts:) on oleg244-server
18:47:36 (1763336856) shut down
facet: mds1 facet_host: oleg244-server facet_failover_host: oleg244-server
Failover mds1 to oleg244-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg244-server: oleg244-server.virtnet: executing set_default_debug -1 all
pdsh@oleg244-client: oleg244-server: ssh exited with exit code 1
Started lustre-MDT0000
18:47:56 (1763336876) targets are mounted
18:47:56 (1763336876) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg244-server
Stopping /mnt/lustre-mds1 (opts:) on oleg244-server
18:48:01 (1763336881) shut down
facet: mds1 facet_host: oleg244-server facet_failover_host: oleg244-server
Failover mds1 to oleg244-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg244-server: oleg244-server.virtnet: executing set_default_debug -1 all
pdsh@oleg244-client: oleg244-server: ssh exited with exit code 1
Started lustre-MDT0000
18:48:25 (1763336905) targets are mounted
18:48:25 (1763336905) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg244-server
Stopping /mnt/lustre-mds1 (opts:) on oleg244-server
18:48:37 (1763336917) shut down
facet: mds1 facet_host: oleg244-server facet_failover_host: oleg244-server
Failover mds1 to oleg244-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg244-server: oleg244-server.virtnet: executing set_default_debug -1 all
pdsh@oleg244-client: oleg244-server: ssh exited with exit code 1
Started lustre-MDT0000
18:48:58 (1763336938) targets are mounted
18:48:58 (1763336938) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg244-server
Stopping /mnt/lustre-mds1 (opts:) on oleg244-server
18:49:14 (1763336954) shut down
facet: mds1 facet_host: oleg244-server facet_failover_host: oleg244-server
Failover mds1 to oleg244-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg244-server: oleg244-server.virtnet: executing set_default_debug -1 all
pdsh@oleg244-client: oleg244-server: ssh exited with exit code 1
Started lustre-MDT0000
18:49:45 (1763336985) targets are mounted
18:49:45 (1763336985) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg244-server
Stopping /mnt/lustre-mds1 (opts:) on oleg244-server
18:50:11 (1763337011) shut down
facet: mds1 facet_host: oleg244-server facet_failover_host: oleg244-server
Failover mds1 to oleg244-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg244-server: oleg244-server.virtnet: executing set_default_debug -1 all
pdsh@oleg244-client: oleg244-server: ssh exited with exit code 1
Started lustre-MDT0000
18:50:30 (1763337030) targets are mounted
18:50:30 (1763337030) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg244-server
Stopping /mnt/lustre-mds1 (opts:) on oleg244-server
18:51:01 (1763337061) shut down
facet: mds1 facet_host: oleg244-server facet_failover_host: oleg244-server
Failover mds1 to oleg244-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg244-server: oleg244-server.virtnet: executing set_default_debug -1 all
pdsh@oleg244-client: oleg244-server: ssh exited with exit code 1
Started lustre-MDT0000
18:51:22 (1763337082) targets are mounted
18:51:22 (1763337082) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg244-server
Stopping /mnt/lustre-mds1 (opts:) on oleg244-server
18:51:58 (1763337118) shut down
facet: mds1 facet_host: oleg244-server facet_failover_host: oleg244-server
Failover mds1 to oleg244-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg244-server: oleg244-server.virtnet: executing set_default_debug -1 all
pdsh@oleg244-client: oleg244-server: ssh exited with exit code 1
Started lustre-MDT0000
18:52:18 (1763337138) targets are mounted
18:52:18 (1763337138) facet_failover done
writemany returned 0