== recovery-small test 51: failover MDS during recovery == 11:51:24 (1780329084)
fail_loc=0x00001310
Failing mds1 on oleg307-server
Stopping /mnt/lustre-mds1 (opts:) on oleg307-server
11:51:28 (1780329088) shut down
facet: mds1 facet_host: oleg307-server facet_failover_host: oleg307-server
Failover mds1 to oleg307-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg307-server: oleg307-server.virtnet: executing set_default_debug -1 all
pdsh@oleg307-client: oleg307-server: ssh exited with exit code 1
Started lustre-MDT0000
11:51:43 (1780329103) targets are mounted
11:51:43 (1780329103) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg307-server
Stopping /mnt/lustre-mds1 (opts:) on oleg307-server
11:51:48 (1780329108) shut down
facet: mds1 facet_host: oleg307-server facet_failover_host: oleg307-server
Failover mds1 to oleg307-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg307-server: oleg307-server.virtnet: executing set_default_debug -1 all
pdsh@oleg307-client: oleg307-server: ssh exited with exit code 1
Started lustre-MDT0000
11:52:04 (1780329124) targets are mounted
11:52:04 (1780329124) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg307-server
Stopping /mnt/lustre-mds1 (opts:) on oleg307-server
11:52:13 (1780329133) shut down
facet: mds1 facet_host: oleg307-server facet_failover_host: oleg307-server
Failover mds1 to oleg307-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg307-server: oleg307-server.virtnet: executing set_default_debug -1 all
pdsh@oleg307-client: oleg307-server: ssh exited with exit code 1
Started lustre-MDT0000
11:52:28 (1780329148) targets are mounted
11:52:28 (1780329148) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg307-server
Stopping /mnt/lustre-mds1 (opts:) on oleg307-server
11:52:42 (1780329162) shut down
facet: mds1 facet_host: oleg307-server facet_failover_host: oleg307-server
Failover mds1 to oleg307-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg307-server: oleg307-server.virtnet: executing set_default_debug -1 all
pdsh@oleg307-client: oleg307-server: ssh exited with exit code 1
Started lustre-MDT0000
11:52:58 (1780329178) targets are mounted
11:52:58 (1780329178) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg307-server
Stopping /mnt/lustre-mds1 (opts:) on oleg307-server
11:53:21 (1780329201) shut down
facet: mds1 facet_host: oleg307-server facet_failover_host: oleg307-server
Failover mds1 to oleg307-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg307-server: oleg307-server.virtnet: executing set_default_debug -1 all
pdsh@oleg307-client: oleg307-server: ssh exited with exit code 1
Started lustre-MDT0000
11:53:36 (1780329216) targets are mounted
11:53:36 (1780329216) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg307-server
Stopping /mnt/lustre-mds1 (opts:) on oleg307-server
11:54:04 (1780329244) shut down
facet: mds1 facet_host: oleg307-server facet_failover_host: oleg307-server
Failover mds1 to oleg307-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg307-server: oleg307-server.virtnet: executing set_default_debug -1 all
pdsh@oleg307-client: oleg307-server: ssh exited with exit code 1
Started lustre-MDT0000
11:54:20 (1780329260) targets are mounted
11:54:20 (1780329260) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg307-server
Stopping /mnt/lustre-mds1 (opts:) on oleg307-server
11:54:53 (1780329293) shut down
facet: mds1 facet_host: oleg307-server facet_failover_host: oleg307-server
Failover mds1 to oleg307-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  lustre-mdt1/mdt1 /mnt/lustre-mds1
oleg307-server: oleg307-server.virtnet: executing set_default_debug -1 all
pdsh@oleg307-client: oleg307-server: ssh exited with exit code 1
Started lustre-MDT0000
11:55:07 (1780329307) targets are mounted
11:55:07 (1780329307) facet_failover done
writemany returned 0