== recovery-small test 51: failover MDS during recovery == 02:58:41 (1786949921)
fail_loc=0x00001310
Failing mds1 on oleg221-server
Stopping /mnt/lustre-mds1 (opts:) on oleg221-server
02:58:49 (1786949929) shut down
facet: mds1 facet_host: oleg221-server facet_failover_host: oleg221-server
Failover mds1 to oleg221-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg221-server: oleg221-server.virtnet: executing set_default_debug -1 all
pdsh@oleg221-client: oleg221-server: ssh exited with exit code 1
Started lustre-MDT0000
02:59:14 (1786949954) targets are mounted
02:59:14 (1786949954) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg221-server
Stopping /mnt/lustre-mds1 (opts:) on oleg221-server
02:59:23 (1786949963) shut down
facet: mds1 facet_host: oleg221-server facet_failover_host: oleg221-server
Failover mds1 to oleg221-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg221-server: oleg221-server.virtnet: executing set_default_debug -1 all
pdsh@oleg221-client: oleg221-server: ssh exited with exit code 1
Started lustre-MDT0000
02:59:56 (1786949996) targets are mounted
02:59:56 (1786949996) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg221-server
Stopping /mnt/lustre-mds1 (opts:) on oleg221-server
03:00:09 (1786950009) shut down
facet: mds1 facet_host: oleg221-server facet_failover_host: oleg221-server
Failover mds1 to oleg221-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg221-server: oleg221-server.virtnet: executing set_default_debug -1 all
pdsh@oleg221-client: oleg221-server: ssh exited with exit code 1
Started lustre-MDT0000
03:00:43 (1786950043) targets are mounted
03:00:43 (1786950043) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg221-server
Stopping /mnt/lustre-mds1 (opts:) on oleg221-server
03:01:02 (1786950062) shut down
facet: mds1 facet_host: oleg221-server facet_failover_host: oleg221-server
Failover mds1 to oleg221-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg221-server: oleg221-server.virtnet: executing set_default_debug -1 all
pdsh@oleg221-client: oleg221-server: ssh exited with exit code 1
Started lustre-MDT0000
03:01:35 (1786950095) targets are mounted
03:01:35 (1786950095) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg221-server
Stopping /mnt/lustre-mds1 (opts:) on oleg221-server
03:02:05 (1786950125) shut down
facet: mds1 facet_host: oleg221-server facet_failover_host: oleg221-server
Failover mds1 to oleg221-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg221-server: oleg221-server.virtnet: executing set_default_debug -1 all
pdsh@oleg221-client: oleg221-server: ssh exited with exit code 1
Started lustre-MDT0000
03:02:39 (1786950159) targets are mounted
03:02:39 (1786950159) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg221-server
Stopping /mnt/lustre-mds1 (opts:) on oleg221-server
03:03:12 (1786950192) shut down
facet: mds1 facet_host: oleg221-server facet_failover_host: oleg221-server
Failover mds1 to oleg221-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg221-server: oleg221-server.virtnet: executing set_default_debug -1 all
pdsh@oleg221-client: oleg221-server: ssh exited with exit code 1
Started lustre-MDT0000
03:03:46 (1786950226) targets are mounted
03:03:47 (1786950227) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg221-server
Stopping /mnt/lustre-mds1 (opts:) on oleg221-server
03:04:26 (1786950266) shut down
facet: mds1 facet_host: oleg221-server facet_failover_host: oleg221-server
Failover mds1 to oleg221-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg221-server: oleg221-server.virtnet: executing set_default_debug -1 all
pdsh@oleg221-client: oleg221-server: ssh exited with exit code 1
Started lustre-MDT0000
03:04:59 (1786950299) targets are mounted
03:04:59 (1786950299) facet_failover done
writemany returned 0