== recovery-small test 51: failover MDS during recovery == 18:24:49 (1775168689)
fail_loc=0x00001310
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
18:24:59 (1775168699) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
18:25:15 (1775168715) targets are mounted
18:25:15 (1775168715) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
18:25:18 (1775168718) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
18:25:34 (1775168734) targets are mounted
18:25:34 (1775168734) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
18:25:41 (1775168741) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
18:25:57 (1775168757) targets are mounted
18:25:57 (1775168757) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
18:26:09 (1775168769) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
18:26:25 (1775168785) targets are mounted
18:26:25 (1775168785) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
18:26:48 (1775168808) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
18:27:03 (1775168823) targets are mounted
18:27:03 (1775168823) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
18:27:30 (1775168850) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
18:27:45 (1775168865) targets are mounted
18:27:45 (1775168865) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg122-server
Stopping /mnt/lustre-mds1 (opts:) on oleg122-server
18:28:17 (1775168897) shut down
facet: mds1 facet_host: oleg122-server facet_failover_host: oleg122-server
Failover mds1 to oleg122-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg122-server: oleg122-server.virtnet: executing set_default_debug -1 all
pdsh@oleg122-client: oleg122-server: ssh exited with exit code 1
Started lustre-MDT0000
18:28:32 (1775168912) targets are mounted
18:28:32 (1775168912) facet_failover done
writemany returned 0