== recovery-small test 51: failover MDS during recovery == 10:49:54 (1788360594)
fail_loc=0x00001310
Failing mds1 on oleg203-server
Stopping /mnt/lustre-mds1 (opts:) on oleg203-server
10:50:03 (1788360603) shut down
facet: mds1 facet_host: oleg203-server facet_failover_host: oleg203-server
Failover mds1 to oleg203-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg203-server: oleg203-server.virtnet: executing set_default_debug -1 all
pdsh@oleg203-client: oleg203-server: ssh exited with exit code 1
Started lustre-MDT0000
10:50:26 (1788360626) targets are mounted
10:50:26 (1788360626) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg203-server
Stopping /mnt/lustre-mds1 (opts:) on oleg203-server
10:50:34 (1788360634) shut down
facet: mds1 facet_host: oleg203-server facet_failover_host: oleg203-server
Failover mds1 to oleg203-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg203-server: oleg203-server.virtnet: executing set_default_debug -1 all
pdsh@oleg203-client: oleg203-server: ssh exited with exit code 1
Started lustre-MDT0000
10:50:55 (1788360655) targets are mounted
10:50:55 (1788360655) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg203-server
Stopping /mnt/lustre-mds1 (opts:) on oleg203-server
10:51:05 (1788360665) shut down
facet: mds1 facet_host: oleg203-server facet_failover_host: oleg203-server
Failover mds1 to oleg203-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg203-server: oleg203-server.virtnet: executing set_default_debug -1 all
pdsh@oleg203-client: oleg203-server: ssh exited with exit code 1
Started lustre-MDT0000
10:51:24 (1788360684) targets are mounted
10:51:24 (1788360684) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg203-server
Stopping /mnt/lustre-mds1 (opts:) on oleg203-server
10:51:38 (1788360698) shut down
facet: mds1 facet_host: oleg203-server facet_failover_host: oleg203-server
Failover mds1 to oleg203-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg203-server: oleg203-server.virtnet: executing set_default_debug -1 all
pdsh@oleg203-client: oleg203-server: ssh exited with exit code 1
Started lustre-MDT0000
10:51:56 (1788360716) targets are mounted
10:51:56 (1788360716) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg203-server
Stopping /mnt/lustre-mds1 (opts:) on oleg203-server
10:52:20 (1788360740) shut down
facet: mds1 facet_host: oleg203-server facet_failover_host: oleg203-server
Failover mds1 to oleg203-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg203-server: oleg203-server.virtnet: executing set_default_debug -1 all
pdsh@oleg203-client: oleg203-server: ssh exited with exit code 1
Started lustre-MDT0000
10:52:39 (1788360759) targets are mounted
10:52:39 (1788360759) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg203-server
Stopping /mnt/lustre-mds1 (opts:) on oleg203-server
10:53:08 (1788360788) shut down
facet: mds1 facet_host: oleg203-server facet_failover_host: oleg203-server
Failover mds1 to oleg203-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg203-server: oleg203-server.virtnet: executing set_default_debug -1 all
pdsh@oleg203-client: oleg203-server: ssh exited with exit code 1
Started lustre-MDT0000
10:53:24 (1788360804) targets are mounted
10:53:24 (1788360804) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg203-server
Stopping /mnt/lustre-mds1 (opts:) on oleg203-server
10:53:59 (1788360839) shut down
facet: mds1 facet_host: oleg203-server facet_failover_host: oleg203-server
Failover mds1 to oleg203-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg203-server: oleg203-server.virtnet: executing set_default_debug -1 all
pdsh@oleg203-client: oleg203-server: ssh exited with exit code 1
Started lustre-MDT0000
10:54:17 (1788360857) targets are mounted
10:54:17 (1788360857) facet_failover done
writemany returned 0