== recovery-small test 51: failover MDS during recovery == 14:55:04 (1783968904)
fail_loc=0x00001310
Failing mds1 on oleg227-server
Stopping /mnt/lustre-mds1 (opts:) on oleg227-server
14:55:27 (1783968927) shut down
facet: mds1 facet_host: oleg227-server facet_failover_host: oleg227-server
Failover mds1 to oleg227-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg227-server: oleg227-server.virtnet: executing set_default_debug -1 all
pdsh@oleg227-client: oleg227-server: ssh exited with exit code 1
Started lustre-MDT0000
14:56:02 (1783968962) targets are mounted
14:56:02 (1783968962) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg227-server
Stopping /mnt/lustre-mds1 (opts:) on oleg227-server
14:56:10 (1783968970) shut down
facet: mds1 facet_host: oleg227-server facet_failover_host: oleg227-server
Failover mds1 to oleg227-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg227-server: oleg227-server.virtnet: executing set_default_debug -1 all
pdsh@oleg227-client: oleg227-server: ssh exited with exit code 1
Started lustre-MDT0000
14:56:36 (1783968996) targets are mounted
14:56:36 (1783968996) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg227-server
Stopping /mnt/lustre-mds1 (opts:) on oleg227-server
14:56:49 (1783969009) shut down
facet: mds1 facet_host: oleg227-server facet_failover_host: oleg227-server
Failover mds1 to oleg227-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg227-server: oleg227-server.virtnet: executing set_default_debug -1 all
pdsh@oleg227-client: oleg227-server: ssh exited with exit code 1
Started lustre-MDT0000
14:57:22 (1783969042) targets are mounted
14:57:22 (1783969042) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg227-server
Stopping /mnt/lustre-mds1 (opts:) on oleg227-server
14:57:41 (1783969061) shut down
facet: mds1 facet_host: oleg227-server facet_failover_host: oleg227-server
Failover mds1 to oleg227-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg227-server: oleg227-server.virtnet: executing set_default_debug -1 all
pdsh@oleg227-client: oleg227-server: ssh exited with exit code 1
Started lustre-MDT0000
14:58:17 (1783969097) targets are mounted
14:58:17 (1783969097) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg227-server
Stopping /mnt/lustre-mds1 (opts:) on oleg227-server
14:58:45 (1783969125) shut down
facet: mds1 facet_host: oleg227-server facet_failover_host: oleg227-server
Failover mds1 to oleg227-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg227-server: oleg227-server.virtnet: executing set_default_debug -1 all
pdsh@oleg227-client: oleg227-server: ssh exited with exit code 1
Started lustre-MDT0000
14:59:12 (1783969152) targets are mounted
14:59:12 (1783969152) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg227-server
Stopping /mnt/lustre-mds1 (opts:) on oleg227-server
14:59:46 (1783969186) shut down
facet: mds1 facet_host: oleg227-server facet_failover_host: oleg227-server
Failover mds1 to oleg227-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg227-server: oleg227-server.virtnet: executing set_default_debug -1 all
pdsh@oleg227-client: oleg227-server: ssh exited with exit code 1
Started lustre-MDT0000
15:00:23 (1783969223) targets are mounted
15:00:23 (1783969223) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg227-server
Stopping /mnt/lustre-mds1 (opts:) on oleg227-server
15:01:04 (1783969264) shut down
facet: mds1 facet_host: oleg227-server facet_failover_host: oleg227-server
Failover mds1 to oleg227-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg227-server: oleg227-server.virtnet: executing set_default_debug -1 all
pdsh@oleg227-client: oleg227-server: ssh exited with exit code 1
Started lustre-MDT0000
15:01:38 (1783969298) targets are mounted
15:01:38 (1783969298) facet_failover done
writemany returned 0