== recovery-small test 51: failover MDS during recovery == 11:22:44 (1754407364)
fail_loc=0x00001310
Failing mds1 on oleg126-server
Stopping /mnt/lustre-mds1 (opts:) on oleg126-server
11:22:47 (1754407367) shut down
facet: mds1 facet_host: oleg126-server facet_failover_host: oleg126-server
Failover mds1 to oleg126-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg126-server: oleg126-server.virtnet: executing set_default_debug -1 all
pdsh@oleg126-client: oleg126-server: ssh exited with exit code 1
Started lustre-MDT0000
11:23:02 (1754407382) targets are mounted
11:23:02 (1754407382) facet_failover done
will failover at 1 5 10 20 25 30
test_51: failover in 1 sec
Failing mds1 on oleg126-server
Stopping /mnt/lustre-mds1 (opts:) on oleg126-server
11:23:05 (1754407385) shut down
facet: mds1 facet_host: oleg126-server facet_failover_host: oleg126-server
Failover mds1 to oleg126-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg126-server: oleg126-server.virtnet: executing set_default_debug -1 all
pdsh@oleg126-client: oleg126-server: ssh exited with exit code 1
Started lustre-MDT0000
11:23:19 (1754407399) targets are mounted
11:23:19 (1754407399) facet_failover done
test_51: failover in 5 sec
Failing mds1 on oleg126-server
Stopping /mnt/lustre-mds1 (opts:) on oleg126-server
11:23:26 (1754407406) shut down
facet: mds1 facet_host: oleg126-server facet_failover_host: oleg126-server
Failover mds1 to oleg126-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg126-server: oleg126-server.virtnet: executing set_default_debug -1 all
pdsh@oleg126-client: oleg126-server: ssh exited with exit code 1
Started lustre-MDT0000
11:23:40 (1754407420) targets are mounted
11:23:40 (1754407420) facet_failover done
test_51: failover in 10 sec
Failing mds1 on oleg126-server
Stopping /mnt/lustre-mds1 (opts:) on oleg126-server
11:23:52 (1754407432) shut down
facet: mds1 facet_host: oleg126-server facet_failover_host: oleg126-server
Failover mds1 to oleg126-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg126-server: oleg126-server.virtnet: executing set_default_debug -1 all
pdsh@oleg126-client: oleg126-server: ssh exited with exit code 1
Started lustre-MDT0000
11:24:07 (1754407447) targets are mounted
11:24:07 (1754407447) facet_failover done
test_51: failover in 20 sec
Failing mds1 on oleg126-server
Stopping /mnt/lustre-mds1 (opts:) on oleg126-server
11:24:30 (1754407470) shut down
facet: mds1 facet_host: oleg126-server facet_failover_host: oleg126-server
Failover mds1 to oleg126-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg126-server: oleg126-server.virtnet: executing set_default_debug -1 all
pdsh@oleg126-client: oleg126-server: ssh exited with exit code 1
Started lustre-MDT0000
11:24:45 (1754407485) targets are mounted
11:24:45 (1754407485) facet_failover done
test_51: failover in 25 sec
Failing mds1 on oleg126-server
Stopping /mnt/lustre-mds1 (opts:) on oleg126-server
11:25:13 (1754407513) shut down
facet: mds1 facet_host: oleg126-server facet_failover_host: oleg126-server
Failover mds1 to oleg126-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg126-server: oleg126-server.virtnet: executing set_default_debug -1 all
pdsh@oleg126-client: oleg126-server: ssh exited with exit code 1
Started lustre-MDT0000
11:25:28 (1754407528) targets are mounted
11:25:28 (1754407528) facet_failover done
test_51: failover in 30 sec
Failing mds1 on oleg126-server
Stopping /mnt/lustre-mds1 (opts:) on oleg126-server
11:26:00 (1754407560) shut down
facet: mds1 facet_host: oleg126-server facet_failover_host: oleg126-server
Failover mds1 to oleg126-server
mount facets: mds1
Start mds1: mount -t lustre -o localrecov  /dev/mapper/mds1_flakey /mnt/lustre-mds1
oleg126-server: oleg126-server.virtnet: executing set_default_debug -1 all
pdsh@oleg126-client: oleg126-server: ssh exited with exit code 1
Started lustre-MDT0000
11:26:15 (1754407575) targets are mounted
11:26:15 (1754407575) facet_failover done
writemany returned 0