== recovery-small test 51: failover MDS during recovery == 14:43:27 (1777574607)
fail_loc=0x00001310
Failing mds1 on oleg430-server
Stopping /mnt/lustre-mds1 (opts:) on oleg430-server
14:43:31 (1777574611) shut down
facet: mds1 facet_host: oleg430-server facet_failover_host: oleg430-server
Failover mds1 to oleg430-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg430-server: oleg430-server.virtnet: executing set_default_debug -1 all
pdsh@oleg430-client: oleg430-server: ssh exited with exit code 1
Started lustre-MDT0000
14:43:47 (1777574627) targets are mounted
14:43:47 (1777574627) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg430-server
Stopping /mnt/lustre-mds1 (opts:) on oleg430-server
14:43:51 (1777574631) shut down
facet: mds1 facet_host: oleg430-server facet_failover_host: oleg430-server
Failover mds1 to oleg430-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg430-server: oleg430-server.virtnet: executing set_default_debug -1 all
pdsh@oleg430-client: oleg430-server: ssh exited with exit code 1
Started lustre-MDT0000
14:44:07 (1777574647) targets are mounted
14:44:07 (1777574647) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg430-server
Stopping /mnt/lustre-mds1 (opts:) on oleg430-server
14:44:14 (1777574654) shut down
facet: mds1 facet_host: oleg430-server facet_failover_host: oleg430-server
Failover mds1 to oleg430-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg430-server: oleg430-server.virtnet: executing set_default_debug -1 all
pdsh@oleg430-client: oleg430-server: ssh exited with exit code 1
Started lustre-MDT0000
14:44:31 (1777574671) targets are mounted
14:44:31 (1777574671) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg430-server
Stopping /mnt/lustre-mds1 (opts:) on oleg430-server
14:44:43 (1777574683) shut down
facet: mds1 facet_host: oleg430-server facet_failover_host: oleg430-server
Failover mds1 to oleg430-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg430-server: oleg430-server.virtnet: executing set_default_debug -1 all
pdsh@oleg430-client: oleg430-server: ssh exited with exit code 1
Started lustre-MDT0000
14:45:00 (1777574700) targets are mounted
14:45:00 (1777574700) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg430-server
Stopping /mnt/lustre-mds1 (opts:) on oleg430-server
14:45:23 (1777574723) shut down
facet: mds1 facet_host: oleg430-server facet_failover_host: oleg430-server
Failover mds1 to oleg430-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg430-server: oleg430-server.virtnet: executing set_default_debug -1 all
pdsh@oleg430-client: oleg430-server: ssh exited with exit code 1
Started lustre-MDT0000
14:45:47 (1777574747) targets are mounted
14:45:47 (1777574747) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg430-server
Stopping /mnt/lustre-mds1 (opts:) on oleg430-server
14:46:19 (1777574779) shut down
facet: mds1 facet_host: oleg430-server facet_failover_host: oleg430-server
Failover mds1 to oleg430-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg430-server: oleg430-server.virtnet: executing set_default_debug -1 all
pdsh@oleg430-client: oleg430-server: ssh exited with exit code 1
Started lustre-MDT0000
14:46:37 (1777574797) targets are mounted
14:46:37 (1777574797) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg430-server
Stopping /mnt/lustre-mds1 (opts:) on oleg430-server
14:47:11 (1777574831) shut down
facet: mds1 facet_host: oleg430-server facet_failover_host: oleg430-server
Failover mds1 to oleg430-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg430-server: oleg430-server.virtnet: executing set_default_debug -1 all
pdsh@oleg430-client: oleg430-server: ssh exited with exit code 1
Started lustre-MDT0000
14:47:28 (1777574848) targets are mounted
14:47:28 (1777574848) facet_failover done
writemany returned 0