== recovery-small test 51: failover MDS during recovery == 16:53:48 (1780088028)
fail_loc=0x00001310
Failing mds1 on oleg260-server
Stopping /mnt/lustre-mds1 (opts:) on oleg260-server
16:53:52 (1780088032) shut down
facet: mds1 facet_host: oleg260-server facet_failover_host: oleg260-server
Failover mds1 to oleg260-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg260-server: oleg260-server.virtnet: executing set_default_debug -1 all
pdsh@oleg260-client: oleg260-server: ssh exited with exit code 1
Started lustre-MDT0000
16:54:07 (1780088047) targets are mounted
16:54:07 (1780088047) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg260-server
Stopping /mnt/lustre-mds1 (opts:) on oleg260-server
16:54:10 (1780088050) shut down
facet: mds1 facet_host: oleg260-server facet_failover_host: oleg260-server
Failover mds1 to oleg260-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg260-server: oleg260-server.virtnet: executing set_default_debug -1 all
pdsh@oleg260-client: oleg260-server: ssh exited with exit code 1
Started lustre-MDT0000
16:54:26 (1780088066) targets are mounted
16:54:26 (1780088066) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg260-server
Stopping /mnt/lustre-mds1 (opts:) on oleg260-server
16:54:33 (1780088073) shut down
facet: mds1 facet_host: oleg260-server facet_failover_host: oleg260-server
Failover mds1 to oleg260-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg260-server: oleg260-server.virtnet: executing set_default_debug -1 all
pdsh@oleg260-client: oleg260-server: ssh exited with exit code 1
Started lustre-MDT0000
16:54:48 (1780088088) targets are mounted
16:54:48 (1780088088) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg260-server
Stopping /mnt/lustre-mds1 (opts:) on oleg260-server
16:55:00 (1780088100) shut down
facet: mds1 facet_host: oleg260-server facet_failover_host: oleg260-server
Failover mds1 to oleg260-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg260-server: oleg260-server.virtnet: executing set_default_debug -1 all
pdsh@oleg260-client: oleg260-server: ssh exited with exit code 1
Started lustre-MDT0000
16:55:15 (1780088115) targets are mounted
16:55:15 (1780088115) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg260-server
Stopping /mnt/lustre-mds1 (opts:) on oleg260-server
16:55:37 (1780088137) shut down
facet: mds1 facet_host: oleg260-server facet_failover_host: oleg260-server
Failover mds1 to oleg260-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg260-server: oleg260-server.virtnet: executing set_default_debug -1 all
pdsh@oleg260-client: oleg260-server: ssh exited with exit code 1
Started lustre-MDT0000
16:55:52 (1780088152) targets are mounted
16:55:52 (1780088152) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg260-server
Stopping /mnt/lustre-mds1 (opts:) on oleg260-server
16:56:19 (1780088179) shut down
facet: mds1 facet_host: oleg260-server facet_failover_host: oleg260-server
Failover mds1 to oleg260-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg260-server: oleg260-server.virtnet: executing set_default_debug -1 all
pdsh@oleg260-client: oleg260-server: ssh exited with exit code 1
Started lustre-MDT0000
16:56:34 (1780088194) targets are mounted
16:56:34 (1780088194) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg260-server
Stopping /mnt/lustre-mds1 (opts:) on oleg260-server
16:57:06 (1780088226) shut down
facet: mds1 facet_host: oleg260-server facet_failover_host: oleg260-server
Failover mds1 to oleg260-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg260-server: oleg260-server.virtnet: executing set_default_debug -1 all
pdsh@oleg260-client: oleg260-server: ssh exited with exit code 1
Started lustre-MDT0000
16:57:20 (1780088240) targets are mounted
16:57:20 (1780088240) facet_failover done
writemany returned 0