== recovery-small test 51: failover MDS during recovery == 07:50:54 (1787572254)
fail_loc=0x00001310
Failing mds1 on oleg156-server
Stopping /mnt/lustre-mds1 (opts:) on oleg156-server
07:51:04 (1787572264) shut down
facet: mds1 facet_host: oleg156-server facet_failover_host: oleg156-server
Failover mds1 to oleg156-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg156-server: oleg156-server.virtnet: executing set_default_debug -1 all
pdsh@oleg156-client: oleg156-server: ssh exited with exit code 1
Started lustre-MDT0000
07:51:39 (1787572299) targets are mounted
07:51:39 (1787572299) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg156-server
Stopping /mnt/lustre-mds1 (opts:) on oleg156-server
07:51:47 (1787572307) shut down
facet: mds1 facet_host: oleg156-server facet_failover_host: oleg156-server
Failover mds1 to oleg156-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg156-server: oleg156-server.virtnet: executing set_default_debug -1 all
pdsh@oleg156-client: oleg156-server: ssh exited with exit code 1
Started lustre-MDT0000
07:52:24 (1787572344) targets are mounted
07:52:24 (1787572344) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg156-server
Stopping /mnt/lustre-mds1 (opts:) on oleg156-server
07:52:38 (1787572358) shut down
facet: mds1 facet_host: oleg156-server facet_failover_host: oleg156-server
Failover mds1 to oleg156-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg156-server: oleg156-server.virtnet: executing set_default_debug -1 all
pdsh@oleg156-client: oleg156-server: ssh exited with exit code 1
Started lustre-MDT0000
07:53:03 (1787572383) targets are mounted
07:53:03 (1787572383) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg156-server
Stopping /mnt/lustre-mds1 (opts:) on oleg156-server
07:53:20 (1787572400) shut down
facet: mds1 facet_host: oleg156-server facet_failover_host: oleg156-server
Failover mds1 to oleg156-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg156-server: oleg156-server.virtnet: executing set_default_debug -1 all
pdsh@oleg156-client: oleg156-server: ssh exited with exit code 1
Started lustre-MDT0000
07:53:48 (1787572428) targets are mounted
07:53:48 (1787572428) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg156-server
Stopping /mnt/lustre-mds1 (opts:) on oleg156-server
07:54:15 (1787572455) shut down
facet: mds1 facet_host: oleg156-server facet_failover_host: oleg156-server
Failover mds1 to oleg156-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg156-server: oleg156-server.virtnet: executing set_default_debug -1 all
pdsh@oleg156-client: oleg156-server: ssh exited with exit code 1
Started lustre-MDT0000
07:54:41 (1787572481) targets are mounted
07:54:41 (1787572481) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg156-server
Stopping /mnt/lustre-mds1 (opts:) on oleg156-server
07:55:13 (1787572513) shut down
facet: mds1 facet_host: oleg156-server facet_failover_host: oleg156-server
Failover mds1 to oleg156-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg156-server: oleg156-server.virtnet: executing set_default_debug -1 all
pdsh@oleg156-client: oleg156-server: ssh exited with exit code 1
Started lustre-MDT0000
07:55:50 (1787572550) targets are mounted
07:55:50 (1787572550) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg156-server
Stopping /mnt/lustre-mds1 (opts:) on oleg156-server
07:56:29 (1787572589) shut down
facet: mds1 facet_host: oleg156-server facet_failover_host: oleg156-server
Failover mds1 to oleg156-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg156-server: oleg156-server.virtnet: executing set_default_debug -1 all
pdsh@oleg156-client: oleg156-server: ssh exited with exit code 1
Started lustre-MDT0000
07:57:04 (1787572624) targets are mounted
07:57:04 (1787572624) facet_failover done
writemany returned 0