== recovery-small test 51: failover MDS during recovery == 17:27:15 (1787866035)
fail_loc=0x00001310
Failing mds1 on oleg644-server
Stopping /mnt/lustre-mds1 (opts:) on oleg644-server
17:27:24 (1787866044) shut down
facet: mds1 facet_host: oleg644-server facet_failover_host: oleg644-server
Failover mds1 to oleg644-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg644-server: oleg644-server.virtnet: executing set_default_debug -1 all
pdsh@oleg644-client: oleg644-server: ssh exited with exit code 1
Started lustre-MDT0000
17:27:48 (1787866068) targets are mounted
17:27:48 (1787866068) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg644-server
Stopping /mnt/lustre-mds1 (opts:) on oleg644-server
17:27:56 (1787866076) shut down
facet: mds1 facet_host: oleg644-server facet_failover_host: oleg644-server
Failover mds1 to oleg644-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg644-server: oleg644-server.virtnet: executing set_default_debug -1 all
pdsh@oleg644-client: oleg644-server: ssh exited with exit code 1
Started lustre-MDT0000
17:28:21 (1787866101) targets are mounted
17:28:21 (1787866101) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg644-server
Stopping /mnt/lustre-mds1 (opts:) on oleg644-server
17:28:33 (1787866113) shut down
facet: mds1 facet_host: oleg644-server facet_failover_host: oleg644-server
Failover mds1 to oleg644-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg644-server: oleg644-server.virtnet: executing set_default_debug -1 all
pdsh@oleg644-client: oleg644-server: ssh exited with exit code 1
Started lustre-MDT0000
17:29:08 (1787866148) targets are mounted
17:29:08 (1787866148) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg644-server
Stopping /mnt/lustre-mds1 (opts:) on oleg644-server
17:29:26 (1787866166) shut down
facet: mds1 facet_host: oleg644-server facet_failover_host: oleg644-server
Failover mds1 to oleg644-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg644-server: oleg644-server.virtnet: executing set_default_debug -1 all
pdsh@oleg644-client: oleg644-server: ssh exited with exit code 1
Started lustre-MDT0000
17:29:51 (1787866191) targets are mounted
17:29:51 (1787866191) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg644-server
Stopping /mnt/lustre-mds1 (opts:) on oleg644-server
17:30:19 (1787866219) shut down
facet: mds1 facet_host: oleg644-server facet_failover_host: oleg644-server
Failover mds1 to oleg644-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg644-server: oleg644-server.virtnet: executing set_default_debug -1 all
pdsh@oleg644-client: oleg644-server: ssh exited with exit code 1
Started lustre-MDT0000
17:30:43 (1787866243) targets are mounted
17:30:43 (1787866243) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg644-server
Stopping /mnt/lustre-mds1 (opts:) on oleg644-server
17:31:17 (1787866277) shut down
facet: mds1 facet_host: oleg644-server facet_failover_host: oleg644-server
Failover mds1 to oleg644-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg644-server: oleg644-server.virtnet: executing set_default_debug -1 all
pdsh@oleg644-client: oleg644-server: ssh exited with exit code 1
Started lustre-MDT0000
17:31:49 (1787866309) targets are mounted
17:31:49 (1787866309) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg644-server
Stopping /mnt/lustre-mds1 (opts:) on oleg644-server
17:32:27 (1787866347) shut down
facet: mds1 facet_host: oleg644-server facet_failover_host: oleg644-server
Failover mds1 to oleg644-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg644-server: oleg644-server.virtnet: executing set_default_debug -1 all
pdsh@oleg644-client: oleg644-server: ssh exited with exit code 1
Started lustre-MDT0000
17:32:51 (1787866371) targets are mounted
17:32:51 (1787866371) facet_failover done
writemany returned 0