== recovery-small test 51: failover MDS during recovery == 09:51:31 (1790949091)
fail_loc=0x00001310
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
09:51:40 (1790949100) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
09:52:02 (1790949122) targets are mounted
09:52:02 (1790949122) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
09:52:10 (1790949130) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
09:52:34 (1790949154) targets are mounted
09:52:34 (1790949154) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
09:52:46 (1790949166) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
09:53:23 (1790949203) targets are mounted
09:53:23 (1790949203) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
09:53:42 (1790949222) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
09:54:14 (1790949254) targets are mounted
09:54:14 (1790949254) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
09:54:44 (1790949284) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
09:55:17 (1790949317) targets are mounted
09:55:17 (1790949317) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
09:55:51 (1790949351) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
09:56:22 (1790949382) targets are mounted
09:56:22 (1790949382) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
09:57:01 (1790949421) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
09:57:27 (1790949447) targets are mounted
09:57:27 (1790949447) facet_failover done
writemany returned 0