== recovery-small test 51: failover MDS during recovery == 13:32:45 (1783445565)
fail_loc=0x00001310
Failing mds1 on oleg410-server
Stopping /mnt/lustre-mds1 (opts:) on oleg410-server
13:32:52 (1783445572) shut down
facet: mds1 facet_host: oleg410-server facet_failover_host: oleg410-server
Failover mds1 to oleg410-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg410-server: oleg410-server.virtnet: executing set_default_debug -1 all
pdsh@oleg410-client: oleg410-server: ssh exited with exit code 1
Started lustre-MDT0000
13:33:27 (1783445607) targets are mounted
13:33:27 (1783445607) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg410-server
Stopping /mnt/lustre-mds1 (opts:) on oleg410-server
13:33:41 (1783445621) shut down
facet: mds1 facet_host: oleg410-server facet_failover_host: oleg410-server
Failover mds1 to oleg410-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg410-server: oleg410-server.virtnet: executing set_default_debug -1 all
pdsh@oleg410-client: oleg410-server: ssh exited with exit code 1
Started lustre-MDT0000
13:34:16 (1783445656) targets are mounted
13:34:16 (1783445656) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg410-server
Stopping /mnt/lustre-mds1 (opts:) on oleg410-server
13:34:29 (1783445669) shut down
facet: mds1 facet_host: oleg410-server facet_failover_host: oleg410-server
Failover mds1 to oleg410-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg410-server: oleg410-server.virtnet: executing set_default_debug -1 all
pdsh@oleg410-client: oleg410-server: ssh exited with exit code 1
Started lustre-MDT0000
13:35:02 (1783445702) targets are mounted
13:35:02 (1783445702) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg410-server
Stopping /mnt/lustre-mds1 (opts:) on oleg410-server
13:35:23 (1783445723) shut down
facet: mds1 facet_host: oleg410-server facet_failover_host: oleg410-server
Failover mds1 to oleg410-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg410-server: oleg410-server.virtnet: executing set_default_debug -1 all
pdsh@oleg410-client: oleg410-server: ssh exited with exit code 1
Started lustre-MDT0000
13:35:56 (1783445756) targets are mounted
13:35:56 (1783445756) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg410-server
Stopping /mnt/lustre-mds1 (opts:) on oleg410-server
13:36:27 (1783445787) shut down
facet: mds1 facet_host: oleg410-server facet_failover_host: oleg410-server
Failover mds1 to oleg410-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg410-server: oleg410-server.virtnet: executing set_default_debug -1 all
pdsh@oleg410-client: oleg410-server: ssh exited with exit code 1
Started lustre-MDT0000
13:36:54 (1783445814) targets are mounted
13:36:54 (1783445814) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg410-server
Stopping /mnt/lustre-mds1 (opts:) on oleg410-server
13:37:32 (1783445852) shut down
facet: mds1 facet_host: oleg410-server facet_failover_host: oleg410-server
Failover mds1 to oleg410-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg410-server: oleg410-server.virtnet: executing set_default_debug -1 all
pdsh@oleg410-client: oleg410-server: ssh exited with exit code 1
Started lustre-MDT0000
13:38:08 (1783445888) targets are mounted
13:38:08 (1783445888) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg410-server
Stopping /mnt/lustre-mds1 (opts:) on oleg410-server
13:38:46 (1783445926) shut down
facet: mds1 facet_host: oleg410-server facet_failover_host: oleg410-server
Failover mds1 to oleg410-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg410-server: oleg410-server.virtnet: executing set_default_debug -1 all
pdsh@oleg410-client: oleg410-server: ssh exited with exit code 1
Started lustre-MDT0000
13:39:20 (1783445960) targets are mounted
13:39:20 (1783445960) facet_failover done
writemany returned 0