== recovery-small test 51: failover MDS during recovery == 23:13:43 (1788837223)
fail_loc=0x00001310
Failing mds1 on oleg444-server
Stopping /mnt/lustre-mds1 (opts:) on oleg444-server
23:13:52 (1788837232) shut down
facet: mds1 facet_host: oleg444-server facet_failover_host: oleg444-server
Failover mds1 to oleg444-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg444-server: oleg444-server.virtnet: executing set_default_debug -1 all
pdsh@oleg444-client: oleg444-server: ssh exited with exit code 1
Started lustre-MDT0000
23:14:25 (1788837265) targets are mounted
23:14:25 (1788837265) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg444-server
Stopping /mnt/lustre-mds1 (opts:) on oleg444-server
23:14:34 (1788837274) shut down
facet: mds1 facet_host: oleg444-server facet_failover_host: oleg444-server
Failover mds1 to oleg444-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg444-server: oleg444-server.virtnet: executing set_default_debug -1 all
pdsh@oleg444-client: oleg444-server: ssh exited with exit code 1
Started lustre-MDT0000
23:14:59 (1788837299) targets are mounted
23:14:59 (1788837299) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg444-server
Stopping /mnt/lustre-mds1 (opts:) on oleg444-server
23:15:11 (1788837311) shut down
facet: mds1 facet_host: oleg444-server facet_failover_host: oleg444-server
Failover mds1 to oleg444-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg444-server: oleg444-server.virtnet: executing set_default_debug -1 all
pdsh@oleg444-client: oleg444-server: ssh exited with exit code 1
Started lustre-MDT0000
23:15:35 (1788837335) targets are mounted
23:15:35 (1788837335) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg444-server
Stopping /mnt/lustre-mds1 (opts:) on oleg444-server
23:15:55 (1788837355) shut down
facet: mds1 facet_host: oleg444-server facet_failover_host: oleg444-server
Failover mds1 to oleg444-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg444-server: oleg444-server.virtnet: executing set_default_debug -1 all
pdsh@oleg444-client: oleg444-server: ssh exited with exit code 1
Started lustre-MDT0000
23:16:28 (1788837388) targets are mounted
23:16:28 (1788837388) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg444-server
Stopping /mnt/lustre-mds1 (opts:) on oleg444-server
23:16:59 (1788837419) shut down
facet: mds1 facet_host: oleg444-server facet_failover_host: oleg444-server
Failover mds1 to oleg444-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg444-server: oleg444-server.virtnet: executing set_default_debug -1 all
pdsh@oleg444-client: oleg444-server: ssh exited with exit code 1
Started lustre-MDT0000
23:17:24 (1788837444) targets are mounted
23:17:24 (1788837444) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg444-server
Stopping /mnt/lustre-mds1 (opts:) on oleg444-server
23:17:58 (1788837478) shut down
facet: mds1 facet_host: oleg444-server facet_failover_host: oleg444-server
Failover mds1 to oleg444-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg444-server: oleg444-server.virtnet: executing set_default_debug -1 all
pdsh@oleg444-client: oleg444-server: ssh exited with exit code 1
Started lustre-MDT0000
23:18:32 (1788837512) targets are mounted
23:18:32 (1788837512) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg444-server
Stopping /mnt/lustre-mds1 (opts:) on oleg444-server
23:19:10 (1788837550) shut down
facet: mds1 facet_host: oleg444-server facet_failover_host: oleg444-server
Failover mds1 to oleg444-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg444-server: oleg444-server.virtnet: executing set_default_debug -1 all
pdsh@oleg444-client: oleg444-server: ssh exited with exit code 1
Started lustre-MDT0000
23:19:36 (1788837576) targets are mounted
23:19:36 (1788837576) facet_failover done
writemany returned 0