== recovery-small test 51: failover MDS during recovery == 22:24:57 (1787192697)
fail_loc=0x00001310
Failing mds1 on oleg407-server
Stopping /mnt/lustre-mds1 (opts:) on oleg407-server
22:25:07 (1787192707) shut down
facet: mds1 facet_host: oleg407-server facet_failover_host: oleg407-server
Failover mds1 to oleg407-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg407-server: oleg407-server.virtnet: executing set_default_debug -1 all
pdsh@oleg407-client: oleg407-server: ssh exited with exit code 1
Started lustre-MDT0000
22:25:42 (1787192742) targets are mounted
22:25:42 (1787192742) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg407-server
Stopping /mnt/lustre-mds1 (opts:) on oleg407-server
22:25:53 (1787192753) shut down
facet: mds1 facet_host: oleg407-server facet_failover_host: oleg407-server
Failover mds1 to oleg407-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg407-server: oleg407-server.virtnet: executing set_default_debug -1 all
pdsh@oleg407-client: oleg407-server: ssh exited with exit code 1
Started lustre-MDT0000
22:26:30 (1787192790) targets are mounted
22:26:30 (1787192790) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg407-server
Stopping /mnt/lustre-mds1 (opts:) on oleg407-server
22:26:44 (1787192804) shut down
facet: mds1 facet_host: oleg407-server facet_failover_host: oleg407-server
Failover mds1 to oleg407-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg407-server: oleg407-server.virtnet: executing set_default_debug -1 all
pdsh@oleg407-client: oleg407-server: ssh exited with exit code 1
Started lustre-MDT0000
22:27:21 (1787192841) targets are mounted
22:27:21 (1787192841) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg407-server
Stopping /mnt/lustre-mds1 (opts:) on oleg407-server
22:27:39 (1787192859) shut down
facet: mds1 facet_host: oleg407-server facet_failover_host: oleg407-server
Failover mds1 to oleg407-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg407-server: oleg407-server.virtnet: executing set_default_debug -1 all
pdsh@oleg407-client: oleg407-server: ssh exited with exit code 1
Started lustre-MDT0000
22:28:04 (1787192884) targets are mounted
22:28:04 (1787192884) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg407-server
Stopping /mnt/lustre-mds1 (opts:) on oleg407-server
22:28:34 (1787192914) shut down
facet: mds1 facet_host: oleg407-server facet_failover_host: oleg407-server
Failover mds1 to oleg407-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg407-server: oleg407-server.virtnet: executing set_default_debug -1 all
pdsh@oleg407-client: oleg407-server: ssh exited with exit code 1
Started lustre-MDT0000
22:29:12 (1787192952) targets are mounted
22:29:12 (1787192952) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg407-server
Stopping /mnt/lustre-mds1 (opts:) on oleg407-server
22:29:46 (1787192986) shut down
facet: mds1 facet_host: oleg407-server facet_failover_host: oleg407-server
Failover mds1 to oleg407-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg407-server: oleg407-server.virtnet: executing set_default_debug -1 all
pdsh@oleg407-client: oleg407-server: ssh exited with exit code 1
Started lustre-MDT0000
22:30:25 (1787193025) targets are mounted
22:30:25 (1787193025) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg407-server
Stopping /mnt/lustre-mds1 (opts:) on oleg407-server
22:31:05 (1787193065) shut down
facet: mds1 facet_host: oleg407-server facet_failover_host: oleg407-server
Failover mds1 to oleg407-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg407-server: oleg407-server.virtnet: executing set_default_debug -1 all
pdsh@oleg407-client: oleg407-server: ssh exited with exit code 1
Started lustre-MDT0000
22:31:44 (1787193104) targets are mounted
22:31:44 (1787193104) facet_failover done
writemany returned 0