== recovery-small test 51: failover MDS during recovery == 07:13:08 (1789125188)
fail_loc=0x00001310
Failing mds1 on oleg217-server
Stopping /mnt/lustre-mds1 (opts:) on oleg217-server
07:13:17 (1789125197) shut down
facet: mds1 facet_host: oleg217-server facet_failover_host: oleg217-server
Failover mds1 to oleg217-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg217-server: oleg217-server.virtnet: executing set_default_debug -1 all
pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1
Started lustre-MDT0000
07:13:52 (1789125232) targets are mounted
07:13:52 (1789125232) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg217-server
Stopping /mnt/lustre-mds1 (opts:) on oleg217-server
07:14:02 (1789125242) shut down
facet: mds1 facet_host: oleg217-server facet_failover_host: oleg217-server
Failover mds1 to oleg217-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg217-server: oleg217-server.virtnet: executing set_default_debug -1 all
pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1
Started lustre-MDT0000
07:14:39 (1789125279) targets are mounted
07:14:39 (1789125279) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg217-server
Stopping /mnt/lustre-mds1 (opts:) on oleg217-server
07:14:53 (1789125293) shut down
facet: mds1 facet_host: oleg217-server facet_failover_host: oleg217-server
Failover mds1 to oleg217-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg217-server: oleg217-server.virtnet: executing set_default_debug -1 all
pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1
Started lustre-MDT0000
07:15:20 (1789125320) targets are mounted
07:15:20 (1789125320) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg217-server
Stopping /mnt/lustre-mds1 (opts:) on oleg217-server
07:15:40 (1789125340) shut down
facet: mds1 facet_host: oleg217-server facet_failover_host: oleg217-server
Failover mds1 to oleg217-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg217-server: oleg217-server.virtnet: executing set_default_debug -1 all
pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1
Started lustre-MDT0000
07:16:11 (1789125371) targets are mounted
07:16:11 (1789125371) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg217-server
Stopping /mnt/lustre-mds1 (opts:) on oleg217-server
07:16:41 (1789125401) shut down
facet: mds1 facet_host: oleg217-server facet_failover_host: oleg217-server
Failover mds1 to oleg217-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg217-server: oleg217-server.virtnet: executing set_default_debug -1 all
pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1
Started lustre-MDT0000
07:17:15 (1789125435) targets are mounted
07:17:15 (1789125435) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg217-server
Stopping /mnt/lustre-mds1 (opts:) on oleg217-server
07:17:51 (1789125471) shut down
facet: mds1 facet_host: oleg217-server facet_failover_host: oleg217-server
Failover mds1 to oleg217-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg217-server: oleg217-server.virtnet: executing set_default_debug -1 all
pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1
Started lustre-MDT0000
07:18:27 (1789125507) targets are mounted
07:18:27 (1789125507) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg217-server
Stopping /mnt/lustre-mds1 (opts:) on oleg217-server
07:19:07 (1789125547) shut down
facet: mds1 facet_host: oleg217-server facet_failover_host: oleg217-server
Failover mds1 to oleg217-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg217-server: oleg217-server.virtnet: executing set_default_debug -1 all
pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1
Started lustre-MDT0000
07:19:40 (1789125580) targets are mounted
07:19:40 (1789125580) facet_failover done
writemany returned 0