== recovery-small test 51: failover MDS during recovery == 11:59:03 (1787846343)
fail_loc=0x00001310
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
11:59:11 (1787846351) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
11:59:43 (1787846383) targets are mounted
11:59:43 (1787846383) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
11:59:51 (1787846391) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
12:00:15 (1787846415) targets are mounted
12:00:15 (1787846415) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
12:00:28 (1787846428) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
12:01:03 (1787846463) targets are mounted
12:01:03 (1787846463) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
12:01:22 (1787846482) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
12:01:55 (1787846515) targets are mounted
12:01:55 (1787846515) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
12:02:23 (1787846543) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
12:02:56 (1787846576) targets are mounted
12:02:56 (1787846576) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
12:03:29 (1787846609) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
12:04:04 (1787846644) targets are mounted
12:04:04 (1787846644) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg628-server
Stopping /mnt/lustre-mds1 (opts:) on oleg628-server
12:04:43 (1787846683) shut down
facet: mds1 facet_host: oleg628-server facet_failover_host: oleg628-server
Failover mds1 to oleg628-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg628-server: oleg628-server.virtnet: executing set_default_debug -1 all
pdsh@oleg628-client: oleg628-server: ssh exited with exit code 1
Started lustre-MDT0000
12:05:17 (1787846717) targets are mounted
12:05:17 (1787846717) facet_failover done
writemany returned 0