== recovery-small test 51: failover MDS during recovery == 01:40:08 (1789018808)
fail_loc=0x00001310
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
01:40:17 (1789018817) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
01:40:52 (1789018852) targets are mounted
01:40:52 (1789018852) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
01:41:02 (1789018862) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
01:41:34 (1789018894) targets are mounted
01:41:34 (1789018894) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
01:41:47 (1789018907) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
01:42:21 (1789018941) targets are mounted
01:42:21 (1789018941) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
01:42:39 (1789018959) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
01:43:14 (1789018994) targets are mounted
01:43:14 (1789018994) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
01:43:43 (1789019023) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
01:44:17 (1789019057) targets are mounted
01:44:17 (1789019057) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
01:44:50 (1789019090) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
01:45:13 (1789019113) targets are mounted
01:45:13 (1789019113) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
01:45:51 (1789019151) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
01:46:14 (1789019174) targets are mounted
01:46:14 (1789019174) facet_failover done
writemany returned 0