== recovery-small test 51: failover MDS during recovery == 16:54:57 (1790974497)
fail_loc=0x00001310
Failing mds1 on oleg131-server
Stopping /mnt/lustre-mds1 (opts:) on oleg131-server
16:55:05 (1790974505) shut down
facet: mds1 facet_host: oleg131-server facet_failover_host: oleg131-server
Failover mds1 to oleg131-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg131-server: oleg131-server.virtnet: executing set_default_debug -1 all
pdsh@oleg131-client: oleg131-server: ssh exited with exit code 1
Started lustre-MDT0000
16:55:38 (1790974538) targets are mounted
16:55:38 (1790974538) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg131-server
Stopping /mnt/lustre-mds1 (opts:) on oleg131-server
16:55:45 (1790974545) shut down
facet: mds1 facet_host: oleg131-server facet_failover_host: oleg131-server
Failover mds1 to oleg131-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg131-server: oleg131-server.virtnet: executing set_default_debug -1 all
pdsh@oleg131-client: oleg131-server: ssh exited with exit code 1
Started lustre-MDT0000
16:56:10 (1790974570) targets are mounted
16:56:10 (1790974570) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg131-server
Stopping /mnt/lustre-mds1 (opts:) on oleg131-server
16:56:22 (1790974582) shut down
facet: mds1 facet_host: oleg131-server facet_failover_host: oleg131-server
Failover mds1 to oleg131-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg131-server: oleg131-server.virtnet: executing set_default_debug -1 all
pdsh@oleg131-client: oleg131-server: ssh exited with exit code 1
Started lustre-MDT0000
16:56:46 (1790974606) targets are mounted
16:56:46 (1790974606) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg131-server
Stopping /mnt/lustre-mds1 (opts:) on oleg131-server
16:57:05 (1790974625) shut down
facet: mds1 facet_host: oleg131-server facet_failover_host: oleg131-server
Failover mds1 to oleg131-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg131-server: oleg131-server.virtnet: executing set_default_debug -1 all
pdsh@oleg131-client: oleg131-server: ssh exited with exit code 1
Started lustre-MDT0000
16:57:38 (1790974658) targets are mounted
16:57:39 (1790974659) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg131-server
Stopping /mnt/lustre-mds1 (opts:) on oleg131-server
16:58:07 (1790974687) shut down
facet: mds1 facet_host: oleg131-server facet_failover_host: oleg131-server
Failover mds1 to oleg131-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg131-server: oleg131-server.virtnet: executing set_default_debug -1 all
pdsh@oleg131-client: oleg131-server: ssh exited with exit code 1
Started lustre-MDT0000
16:58:41 (1790974721) targets are mounted
16:58:41 (1790974721) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg131-server
Stopping /mnt/lustre-mds1 (opts:) on oleg131-server
16:59:14 (1790974754) shut down
facet: mds1 facet_host: oleg131-server facet_failover_host: oleg131-server
Failover mds1 to oleg131-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg131-server: oleg131-server.virtnet: executing set_default_debug -1 all
pdsh@oleg131-client: oleg131-server: ssh exited with exit code 1
Started lustre-MDT0000
16:59:37 (1790974777) targets are mounted
16:59:37 (1790974777) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg131-server
Stopping /mnt/lustre-mds1 (opts:) on oleg131-server
17:00:16 (1790974816) shut down
facet: mds1 facet_host: oleg131-server facet_failover_host: oleg131-server
Failover mds1 to oleg131-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg131-server: oleg131-server.virtnet: executing set_default_debug -1 all
pdsh@oleg131-client: oleg131-server: ssh exited with exit code 1
Started lustre-MDT0000
17:00:41 (1790974841) targets are mounted
17:00:41 (1790974841) facet_failover done
writemany returned 0