== recovery-small test 51: failover MDS during recovery == 03:53:49 (1787558029)
fail_loc=0x00001310
Failing mds1 on oleg619-server
Stopping /mnt/lustre-mds1 (opts:) on oleg619-server
03:53:56 (1787558036) shut down
facet: mds1 facet_host: oleg619-server facet_failover_host: oleg619-server
Failover mds1 to oleg619-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg619-server: oleg619-server.virtnet: executing set_default_debug -1 all
pdsh@oleg619-client: oleg619-server: ssh exited with exit code 1
Started lustre-MDT0000
03:54:19 (1787558059) targets are mounted
03:54:19 (1787558059) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg619-server
Stopping /mnt/lustre-mds1 (opts:) on oleg619-server
03:54:26 (1787558066) shut down
facet: mds1 facet_host: oleg619-server facet_failover_host: oleg619-server
Failover mds1 to oleg619-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg619-server: oleg619-server.virtnet: executing set_default_debug -1 all
pdsh@oleg619-client: oleg619-server: ssh exited with exit code 1
Started lustre-MDT0000
03:54:48 (1787558088) targets are mounted
03:54:48 (1787558088) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg619-server
Stopping /mnt/lustre-mds1 (opts:) on oleg619-server
03:54:59 (1787558099) shut down
facet: mds1 facet_host: oleg619-server facet_failover_host: oleg619-server
Failover mds1 to oleg619-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg619-server: oleg619-server.virtnet: executing set_default_debug -1 all
pdsh@oleg619-client: oleg619-server: ssh exited with exit code 1
Started lustre-MDT0000
03:55:32 (1787558132) targets are mounted
03:55:32 (1787558132) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg619-server
Stopping /mnt/lustre-mds1 (opts:) on oleg619-server
03:55:51 (1787558151) shut down
facet: mds1 facet_host: oleg619-server facet_failover_host: oleg619-server
Failover mds1 to oleg619-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg619-server: oleg619-server.virtnet: executing set_default_debug -1 all
pdsh@oleg619-client: oleg619-server: ssh exited with exit code 1
Started lustre-MDT0000
03:56:23 (1787558183) targets are mounted
03:56:23 (1787558183) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg619-server
Stopping /mnt/lustre-mds1 (opts:) on oleg619-server
03:56:50 (1787558210) shut down
facet: mds1 facet_host: oleg619-server facet_failover_host: oleg619-server
Failover mds1 to oleg619-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg619-server: oleg619-server.virtnet: executing set_default_debug -1 all
pdsh@oleg619-client: oleg619-server: ssh exited with exit code 1
Started lustre-MDT0000
03:57:15 (1787558235) targets are mounted
03:57:15 (1787558235) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg619-server
Stopping /mnt/lustre-mds1 (opts:) on oleg619-server
03:57:48 (1787558268) shut down
facet: mds1 facet_host: oleg619-server facet_failover_host: oleg619-server
Failover mds1 to oleg619-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg619-server: oleg619-server.virtnet: executing set_default_debug -1 all
pdsh@oleg619-client: oleg619-server: ssh exited with exit code 1
Started lustre-MDT0000
03:58:22 (1787558302) targets are mounted
03:58:22 (1787558302) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg619-server
Stopping /mnt/lustre-mds1 (opts:) on oleg619-server
03:59:03 (1787558343) shut down
facet: mds1 facet_host: oleg619-server facet_failover_host: oleg619-server
Failover mds1 to oleg619-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg619-server: oleg619-server.virtnet: executing set_default_debug -1 all
pdsh@oleg619-client: oleg619-server: ssh exited with exit code 1
Started lustre-MDT0000
03:59:29 (1787558369) targets are mounted
03:59:29 (1787558369) facet_failover done
writemany returned 0