== recovery-small test 51: failover MDS during recovery == 11:15:06 (1787584506)
fail_loc=0x00001310
Failing mds1 on oleg201-server
Stopping /mnt/lustre-mds1 (opts:) on oleg201-server
11:15:14 (1787584514) shut down
facet: mds1 facet_host: oleg201-server facet_failover_host: oleg201-server
Failover mds1 to oleg201-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg201-server: oleg201-server.virtnet: executing set_default_debug -1 all
pdsh@oleg201-client: oleg201-server: ssh exited with exit code 1
Started lustre-MDT0000
11:15:39 (1787584539) targets are mounted
11:15:39 (1787584539) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg201-server
Stopping /mnt/lustre-mds1 (opts:) on oleg201-server
11:15:47 (1787584547) shut down
facet: mds1 facet_host: oleg201-server facet_failover_host: oleg201-server
Failover mds1 to oleg201-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg201-server: oleg201-server.virtnet: executing set_default_debug -1 all
pdsh@oleg201-client: oleg201-server: ssh exited with exit code 1
Started lustre-MDT0000
11:16:09 (1787584569) targets are mounted
11:16:09 (1787584569) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg201-server
Stopping /mnt/lustre-mds1 (opts:) on oleg201-server
11:16:21 (1787584581) shut down
facet: mds1 facet_host: oleg201-server facet_failover_host: oleg201-server
Failover mds1 to oleg201-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg201-server: oleg201-server.virtnet: executing set_default_debug -1 all
pdsh@oleg201-client: oleg201-server: ssh exited with exit code 1
Started lustre-MDT0000
11:16:47 (1787584607) targets are mounted
11:16:47 (1787584607) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg201-server
Stopping /mnt/lustre-mds1 (opts:) on oleg201-server
11:17:06 (1787584626) shut down
facet: mds1 facet_host: oleg201-server facet_failover_host: oleg201-server
Failover mds1 to oleg201-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg201-server: oleg201-server.virtnet: executing set_default_debug -1 all
pdsh@oleg201-client: oleg201-server: ssh exited with exit code 1
Started lustre-MDT0000
11:17:40 (1787584660) targets are mounted
11:17:40 (1787584660) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg201-server
Stopping /mnt/lustre-mds1 (opts:) on oleg201-server
11:18:10 (1787584690) shut down
facet: mds1 facet_host: oleg201-server facet_failover_host: oleg201-server
Failover mds1 to oleg201-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg201-server: oleg201-server.virtnet: executing set_default_debug -1 all
pdsh@oleg201-client: oleg201-server: ssh exited with exit code 1
Started lustre-MDT0000
11:18:43 (1787584723) targets are mounted
11:18:43 (1787584723) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg201-server
Stopping /mnt/lustre-mds1 (opts:) on oleg201-server
11:19:18 (1787584758) shut down
facet: mds1 facet_host: oleg201-server facet_failover_host: oleg201-server
Failover mds1 to oleg201-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg201-server: oleg201-server.virtnet: executing set_default_debug -1 all
pdsh@oleg201-client: oleg201-server: ssh exited with exit code 1
Started lustre-MDT0000
11:19:51 (1787584791) targets are mounted
11:19:51 (1787584791) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg201-server
Stopping /mnt/lustre-mds1 (opts:) on oleg201-server
11:20:30 (1787584830) shut down
facet: mds1 facet_host: oleg201-server facet_failover_host: oleg201-server
Failover mds1 to oleg201-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg201-server: oleg201-server.virtnet: executing set_default_debug -1 all
pdsh@oleg201-client: oleg201-server: ssh exited with exit code 1
Started lustre-MDT0000
11:21:05 (1787584865) targets are mounted
11:21:05 (1787584865) facet_failover done
writemany returned 0