== recovery-small test 51: failover MDS during recovery == 05:49:00 (1788688140)
fail_loc=0x00001310
Failing mds1 on oleg641-server
Stopping /mnt/lustre-mds1 (opts:) on oleg641-server
05:49:07 (1788688147) shut down
facet: mds1 facet_host: oleg641-server facet_failover_host: oleg641-server
Failover mds1 to oleg641-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg641-server: oleg641-server.virtnet: executing set_default_debug -1 all
pdsh@oleg641-client: oleg641-server: ssh exited with exit code 1
Started lustre-MDT0000
05:49:41 (1788688181) targets are mounted
05:49:41 (1788688181) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg641-server
Stopping /mnt/lustre-mds1 (opts:) on oleg641-server
05:49:50 (1788688190) shut down
facet: mds1 facet_host: oleg641-server facet_failover_host: oleg641-server
Failover mds1 to oleg641-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg641-server: oleg641-server.virtnet: executing set_default_debug -1 all
pdsh@oleg641-client: oleg641-server: ssh exited with exit code 1
Started lustre-MDT0000
05:50:23 (1788688223) targets are mounted
05:50:23 (1788688223) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg641-server
Stopping /mnt/lustre-mds1 (opts:) on oleg641-server
05:50:35 (1788688235) shut down
facet: mds1 facet_host: oleg641-server facet_failover_host: oleg641-server
Failover mds1 to oleg641-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg641-server: oleg641-server.virtnet: executing set_default_debug -1 all
pdsh@oleg641-client: oleg641-server: ssh exited with exit code 1
Started lustre-MDT0000
05:51:09 (1788688269) targets are mounted
05:51:09 (1788688269) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg641-server
Stopping /mnt/lustre-mds1 (opts:) on oleg641-server
05:51:31 (1788688291) shut down
facet: mds1 facet_host: oleg641-server facet_failover_host: oleg641-server
Failover mds1 to oleg641-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg641-server: oleg641-server.virtnet: executing set_default_debug -1 all
pdsh@oleg641-client: oleg641-server: ssh exited with exit code 1
Started lustre-MDT0000
05:52:02 (1788688322) targets are mounted
05:52:02 (1788688322) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg641-server
Stopping /mnt/lustre-mds1 (opts:) on oleg641-server
05:52:31 (1788688351) shut down
facet: mds1 facet_host: oleg641-server facet_failover_host: oleg641-server
Failover mds1 to oleg641-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg641-server: oleg641-server.virtnet: executing set_default_debug -1 all
pdsh@oleg641-client: oleg641-server: ssh exited with exit code 1
Started lustre-MDT0000
05:53:04 (1788688384) targets are mounted
05:53:04 (1788688384) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg641-server
Stopping /mnt/lustre-mds1 (opts:) on oleg641-server
05:53:37 (1788688417) shut down
facet: mds1 facet_host: oleg641-server facet_failover_host: oleg641-server
Failover mds1 to oleg641-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg641-server: oleg641-server.virtnet: executing set_default_debug -1 all
pdsh@oleg641-client: oleg641-server: ssh exited with exit code 1
Started lustre-MDT0000
05:54:03 (1788688443) targets are mounted
05:54:03 (1788688443) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg641-server
Stopping /mnt/lustre-mds1 (opts:) on oleg641-server
05:54:41 (1788688481) shut down
facet: mds1 facet_host: oleg641-server facet_failover_host: oleg641-server
Failover mds1 to oleg641-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg641-server: oleg641-server.virtnet: executing set_default_debug -1 all
pdsh@oleg641-client: oleg641-server: ssh exited with exit code 1
Started lustre-MDT0000
05:55:15 (1788688515) targets are mounted
05:55:15 (1788688515) facet_failover done
writemany returned 0