== recovery-small test 51: failover MDS during recovery == 03:40:16 (1782632416)
fail_loc=0x00001310
Failing mds1 on oleg359-server
Stopping /mnt/lustre-mds1 (opts:) on oleg359-server
03:40:20 (1782632420) shut down
facet: mds1 facet_host: oleg359-server facet_failover_host: oleg359-server
Failover mds1 to oleg359-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg359-server: oleg359-server.virtnet: executing set_default_debug -1 all
pdsh@oleg359-client: oleg359-server: ssh exited with exit code 1
Started lustre-MDT0000
03:40:35 (1782632435) targets are mounted
03:40:35 (1782632435) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg359-server
Stopping /mnt/lustre-mds1 (opts:) on oleg359-server
03:40:37 (1782632437) shut down
facet: mds1 facet_host: oleg359-server facet_failover_host: oleg359-server
Failover mds1 to oleg359-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg359-server: oleg359-server.virtnet: executing set_default_debug -1 all
pdsh@oleg359-client: oleg359-server: ssh exited with exit code 1
Started lustre-MDT0000
03:40:52 (1782632452) targets are mounted
03:40:52 (1782632452) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg359-server
Stopping /mnt/lustre-mds1 (opts:) on oleg359-server
03:40:59 (1782632459) shut down
facet: mds1 facet_host: oleg359-server facet_failover_host: oleg359-server
Failover mds1 to oleg359-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg359-server: oleg359-server.virtnet: executing set_default_debug -1 all
pdsh@oleg359-client: oleg359-server: ssh exited with exit code 1
Started lustre-MDT0000
03:41:13 (1782632473) targets are mounted
03:41:13 (1782632473) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg359-server
Stopping /mnt/lustre-mds1 (opts:) on oleg359-server
03:41:25 (1782632485) shut down
facet: mds1 facet_host: oleg359-server facet_failover_host: oleg359-server
Failover mds1 to oleg359-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg359-server: oleg359-server.virtnet: executing set_default_debug -1 all
pdsh@oleg359-client: oleg359-server: ssh exited with exit code 1
Started lustre-MDT0000
03:41:40 (1782632500) targets are mounted
03:41:40 (1782632500) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg359-server
Stopping /mnt/lustre-mds1 (opts:) on oleg359-server
03:42:02 (1782632522) shut down
facet: mds1 facet_host: oleg359-server facet_failover_host: oleg359-server
Failover mds1 to oleg359-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg359-server: oleg359-server.virtnet: executing set_default_debug -1 all
pdsh@oleg359-client: oleg359-server: ssh exited with exit code 1
Started lustre-MDT0000
03:42:16 (1782632536) targets are mounted
03:42:16 (1782632536) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg359-server
Stopping /mnt/lustre-mds1 (opts:) on oleg359-server
03:42:44 (1782632564) shut down
facet: mds1 facet_host: oleg359-server facet_failover_host: oleg359-server
Failover mds1 to oleg359-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg359-server: oleg359-server.virtnet: executing set_default_debug -1 all
pdsh@oleg359-client: oleg359-server: ssh exited with exit code 1
Started lustre-MDT0000
03:42:58 (1782632578) targets are mounted
03:42:58 (1782632578) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg359-server
Stopping /mnt/lustre-mds1 (opts:) on oleg359-server
03:43:30 (1782632610) shut down
facet: mds1 facet_host: oleg359-server facet_failover_host: oleg359-server
Failover mds1 to oleg359-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg359-server: oleg359-server.virtnet: executing set_default_debug -1 all
pdsh@oleg359-client: oleg359-server: ssh exited with exit code 1
Started lustre-MDT0000
03:43:45 (1782632625) targets are mounted
03:43:45 (1782632625) facet_failover done
writemany returned 0