== recovery-small test 51: failover MDS during recovery == 01:57:24 (1787551044)
fail_loc=0x00001310
Failing mds1 on oleg111-server
Stopping /mnt/lustre-mds1 (opts:) on oleg111-server
01:57:33 (1787551053) shut down
facet: mds1 facet_host: oleg111-server facet_failover_host: oleg111-server
Failover mds1 to oleg111-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg111-server: oleg111-server.virtnet: executing set_default_debug -1 all
pdsh@oleg111-client: oleg111-server: ssh exited with exit code 1
Started lustre-MDT0000
01:58:11 (1787551091) targets are mounted
01:58:11 (1787551091) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg111-server
Stopping /mnt/lustre-mds1 (opts:) on oleg111-server
01:58:22 (1787551102) shut down
facet: mds1 facet_host: oleg111-server facet_failover_host: oleg111-server
Failover mds1 to oleg111-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg111-server: oleg111-server.virtnet: executing set_default_debug -1 all
pdsh@oleg111-client: oleg111-server: ssh exited with exit code 1
Started lustre-MDT0000
01:58:53 (1787551133) targets are mounted
01:58:53 (1787551133) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg111-server
Stopping /mnt/lustre-mds1 (opts:) on oleg111-server
01:59:07 (1787551147) shut down
facet: mds1 facet_host: oleg111-server facet_failover_host: oleg111-server
Failover mds1 to oleg111-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg111-server: oleg111-server.virtnet: executing set_default_debug -1 all
pdsh@oleg111-client: oleg111-server: ssh exited with exit code 1
Started lustre-MDT0000
01:59:41 (1787551181) targets are mounted
01:59:41 (1787551181) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg111-server
Stopping /mnt/lustre-mds1 (opts:) on oleg111-server
02:00:03 (1787551203) shut down
facet: mds1 facet_host: oleg111-server facet_failover_host: oleg111-server
Failover mds1 to oleg111-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg111-server: oleg111-server.virtnet: executing set_default_debug -1 all
pdsh@oleg111-client: oleg111-server: ssh exited with exit code 1
Started lustre-MDT0000
02:00:41 (1787551241) targets are mounted
02:00:41 (1787551241) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg111-server
Stopping /mnt/lustre-mds1 (opts:) on oleg111-server
02:01:13 (1787551273) shut down
facet: mds1 facet_host: oleg111-server facet_failover_host: oleg111-server
Failover mds1 to oleg111-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg111-server: oleg111-server.virtnet: executing set_default_debug -1 all
pdsh@oleg111-client: oleg111-server: ssh exited with exit code 1
Started lustre-MDT0000
02:01:48 (1787551308) targets are mounted
02:01:48 (1787551308) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg111-server
Stopping /mnt/lustre-mds1 (opts:) on oleg111-server
02:02:25 (1787551345) shut down
facet: mds1 facet_host: oleg111-server facet_failover_host: oleg111-server
Failover mds1 to oleg111-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg111-server: oleg111-server.virtnet: executing set_default_debug -1 all
pdsh@oleg111-client: oleg111-server: ssh exited with exit code 1
Started lustre-MDT0000
02:02:56 (1787551376) targets are mounted
02:02:56 (1787551376) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg111-server
Stopping /mnt/lustre-mds1 (opts:) on oleg111-server
02:03:39 (1787551419) shut down
facet: mds1 facet_host: oleg111-server facet_failover_host: oleg111-server
Failover mds1 to oleg111-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg111-server: oleg111-server.virtnet: executing set_default_debug -1 all
pdsh@oleg111-client: oleg111-server: ssh exited with exit code 1
Started lustre-MDT0000
02:04:15 (1787551455) targets are mounted
02:04:15 (1787551455) facet_failover done
writemany returned 0