== recovery-small test 144a: MDT failover should stop precreation threads ========================================================== 15:18:00 (1772309880) striped dir -i0 -c1 -H all_char /mnt/lustre/d144a.recovery-small timeout=300 Failing ost1 on oleg409-server Stopping /mnt/lustre-ost1 (opts:) on oleg409-server reboot facets: ost1 Failover ost1 to oleg409-server mount facets: ost1 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 oleg409-server: oleg409-server.virtnet: executing set_default_debug -1 all 8 pdsh@oleg409-client: oleg409-server: ssh exited with exit code 1 Started lustre-OST0000 oleg409-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid in FULL state after 0 sec Failing mds1 on oleg409-server Stopping /mnt/lustre-mds1 (opts:) on oleg409-server reboot facets: mds1 Failover mds1 to oleg409-server mount facets: mds1 Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg409-server: oleg409-server.virtnet: executing set_default_debug -1 all 8 pdsh@oleg409-client: oleg409-server: ssh exited with exit code 1 Started lustre-MDT0000 oleg409-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103421 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103422 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103423 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103424 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103425 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103426 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103427 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103428 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103429 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103430 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103431 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103432 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103435 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103433 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103434 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103436 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103437 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103438 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 103439 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 103440 Killed touch $DIR/$tdir/$tfile_$i Failing mds1 on oleg409-server Stopping /mnt/lustre-mds1 (opts:) on oleg409-server /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4330: 103441 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4330: 103442 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103443 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103444 Killed touch $DIR/$tdir/$tfile_$i reboot facets: mds1 Failover mds1 to oleg409-server mount facets: mds1 Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg409-server: oleg409-server.virtnet: executing set_default_debug -1 all 8 pdsh@oleg409-client: oleg409-server: ssh exited with exit code 1 Started lustre-MDT0000 oleg409-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103445 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103446 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103447 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103448 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103449 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103450 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103451 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103452 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103453 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103454 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103455 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4169: 103456 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103457 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103458 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103459 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103460 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3637: 103462 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 103461 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 3615: 103463 Killed touch $DIR/$tdir/$tfile_$i /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4298: 103465 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4274: 103464 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 4274: 103466 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 7591: 103467 Killed touch $DIR/$tdir/$tfile_$i (wd: ~) timeout=20