== recovery-small test 144a: MDT failover should stop precreation threads ========================================================== 15:07:57 (1776193677) timeout=300 Failing ost1 on oleg405-server Stopping /mnt/lustre-ost1 (opts:) on oleg405-server reboot facets: ost1 Failover ost1 to oleg405-server mount facets: ost1 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 oleg405-server: oleg405-server.virtnet: executing set_default_debug -1 all 8 pdsh@oleg405-client: oleg405-server: ssh exited with exit code 1 Started lustre-OST0000 oleg405-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid in FULL state after 0 sec Failing mds1 on oleg405-server Stopping /mnt/lustre-mds1 (opts:) on oleg405-server reboot facets: mds1 Failover mds1 to oleg405-server mount facets: mds1 Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg405-server: oleg405-server.virtnet: executing set_default_debug -1 all 8 pdsh@oleg405-client: oleg405-server: ssh exited with exit code 1 Started lustre-MDT0000 /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 95403 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 95402 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 95405 Killed touch $DIR/$tdir/$tfile_$i oleg405-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95404 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95406 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95407 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95408 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95409 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95410 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95411 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95412 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95414 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95415 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95416 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95417 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95418 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95419 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95420 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 95421 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 95422 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 95413 Killed touch $DIR/$tdir/$tfile_$i Failing mds1 on oleg405-server Stopping /mnt/lustre-mds1 (opts:) on oleg405-server /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4341: 95424 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4341: 95425 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95423 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95426 Killed touch $DIR/$tdir/$tfile_$i reboot facets: mds1 Failover mds1 to oleg405-server mount facets: mds1 Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg405-server: oleg405-server.virtnet: executing set_default_debug -1 all 8 pdsh@oleg405-client: oleg405-server: ssh exited with exit code 1 Started lustre-MDT0000 oleg405-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95427 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95428 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95429 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95430 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95431 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95432 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95433 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95434 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95435 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95436 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95437 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95438 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4180: 95442 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 95440 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 95441 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 95443 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4309: 95444 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4285: 95446 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4285: 95447 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 7602: 95439 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 7602: 95445 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) timeout=20