== recovery-small test 51: failover MDS during recovery == 00:02:06 (1762491726)
fail_loc=0x00001310
Failing mds1 on oleg320-server
Stopping /mnt/lustre-mds1 (opts:) on oleg320-server
00:02:14 (1762491734) shut down
facet: mds1 facet_host: oleg320-server facet_failover_host: oleg320-server
Failover mds1 to oleg320-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg320-server: oleg320-server.virtnet: executing set_default_debug -1 all
pdsh@oleg320-client: oleg320-server: ssh exited with exit code 1
Started lustre-MDT0000
00:02:34 (1762491754) targets are mounted
00:02:34 (1762491754) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg320-server
Stopping /mnt/lustre-mds1 (opts:) on oleg320-server
00:02:46 (1762491766) shut down
facet: mds1 facet_host: oleg320-server facet_failover_host: oleg320-server
Failover mds1 to oleg320-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg320-server: oleg320-server.virtnet: executing set_default_debug -1 all
pdsh@oleg320-client: oleg320-server: ssh exited with exit code 1
Started lustre-MDT0000
00:03:16 (1762491796) targets are mounted
00:03:16 (1762491796) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg320-server
Stopping /mnt/lustre-mds1 (opts:) on oleg320-server
00:03:26 (1762491806) shut down
facet: mds1 facet_host: oleg320-server facet_failover_host: oleg320-server
Failover mds1 to oleg320-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg320-server: oleg320-server.virtnet: executing set_default_debug -1 all
pdsh@oleg320-client: oleg320-server: ssh exited with exit code 1
Started lustre-MDT0000
00:03:46 (1762491826) targets are mounted
00:03:46 (1762491826) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg320-server
Stopping /mnt/lustre-mds1 (opts:) on oleg320-server
00:04:01 (1762491841) shut down
facet: mds1 facet_host: oleg320-server facet_failover_host: oleg320-server
Failover mds1 to oleg320-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg320-server: oleg320-server.virtnet: executing set_default_debug -1 all
pdsh@oleg320-client: oleg320-server: ssh exited with exit code 1
Started lustre-MDT0000
00:04:19 (1762491859) targets are mounted
00:04:19 (1762491859) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg320-server
Stopping /mnt/lustre-mds1 (opts:) on oleg320-server
00:04:44 (1762491884) shut down
facet: mds1 facet_host: oleg320-server facet_failover_host: oleg320-server
Failover mds1 to oleg320-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg320-server: oleg320-server.virtnet: executing set_default_debug -1 all
pdsh@oleg320-client: oleg320-server: ssh exited with exit code 1
Started lustre-MDT0000
00:05:07 (1762491907) targets are mounted
00:05:07 (1762491907) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg320-server
Stopping /mnt/lustre-mds1 (opts:) on oleg320-server
00:05:37 (1762491937) shut down
facet: mds1 facet_host: oleg320-server facet_failover_host: oleg320-server
Failover mds1 to oleg320-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg320-server: oleg320-server.virtnet: executing set_default_debug -1 all
pdsh@oleg320-client: oleg320-server: ssh exited with exit code 1
Started lustre-MDT0000
00:05:58 (1762491958) targets are mounted
00:05:58 (1762491958) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg320-server
Stopping /mnt/lustre-mds1 (opts:) on oleg320-server
00:06:32 (1762491992) shut down
facet: mds1 facet_host: oleg320-server facet_failover_host: oleg320-server
Failover mds1 to oleg320-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg320-server: oleg320-server.virtnet: executing set_default_debug -1 all
pdsh@oleg320-client: oleg320-server: ssh exited with exit code 1
Started lustre-MDT0000
00:06:55 (1762492015) targets are mounted
00:06:55 (1762492015) facet_failover done
writemany returned 0