== sanity-lnet test 211: Remote NI recovery checks ======= 01:21:41 (1788844901) /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 default via 192.168.202.254 dev ens2 192.168.202.0/24 dev ens2 proto kernel scope link src 192.168.202.29 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 default via 192.168.202.254 dev ens2 192.168.202.0/24 dev ens2 proto kernel scope link src 192.168.202.29 default via 192.168.202.254 dev ens2 192.168.202.0/24 dev ens2 proto kernel scope link src 192.168.202.29 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.202.29@tcp discover: - primary nid: 192.168.202.29@tcp Multi-Rail: true peer_ni: - nid: 192.168.202.29@tcp - nid: 192.168.202.29@tcp1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/utils/lctl net_drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -r 1 /home/green/git/lustre-release/lustre/utils/lctl net_drop add -s *@tcp1 -d *@tcp1 -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.202.29@tcp --health 0 -p recovery queue should have 192.168.202.29@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p Peer NI recovery: nid-0: 192.168.202.29@tcp -p recovery queue should be empty Waiting 20s for '0' Waiting 10s for '0' Updated after 15s: want '0' got '0' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p Check ping counts: - nid: 192.168.202.29@tcp ping_count: 0 - nid: 192.168.202.29@tcp1 ping_count: 0 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop del -a /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.202.29@tcp --health 500 -p recovery queue should have 192.168.202.29@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p Peer NI recovery: nid-0: 192.168.202.29@tcp -p recovery queue should be empty Waiting 20s for '0' Updated after 7s: want '0' got '0' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 default via 192.168.202.254 dev ens2 192.168.202.0/24 dev ens2 proto kernel scope link src 192.168.202.29 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 default via 192.168.202.254 dev ens2 192.168.202.0/24 dev ens2 proto kernel scope link src 192.168.202.29 default via 192.168.202.254 dev ens2 192.168.202.0/24 dev ens2 proto kernel scope link src 192.168.202.29 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.202.29@tcp discover: - primary nid: 192.168.202.29@tcp Multi-Rail: true peer_ni: - nid: 192.168.202.29@tcp - nid: 192.168.202.29@tcp1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/utils/lctl net_drop add -s *@tcp -d *@tcp -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.202.29@tcp --health 0 Check ping counts: Waiting 4s for '1' Update not seen after 4s: want '1' got '2' - nid: 192.168.202.29@tcp ping_count: 2 - nid: 192.168.202.29@tcp1 ping_count: 0 sanity-lnet test_211: @@@@@@ FAIL: Unexpected ping count Trace dump: = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7524:error() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:3516:check_ping_count() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:3675:test_211() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7901:run_one() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7964:run_one_logged() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7778:run_test() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:3690:main() Dumping lctl log to /tmp/testlogs//sanity-lnet.test_211.*.1788844940.log oleg229-server: invalid parameter 'dump_kernel' oleg229-server: open(dump_kernel) failed: No such file or directory rsync: chown "/tmp/testlogs/.sanity-lnet.test_211.debug_log.oleg229-server.1788844940.log.6A9v9i" failed: Operation not permitted (1) rsync: chown "/tmp/testlogs/.sanity-lnet.test_211.dmesg.oleg229-server.1788844940.log.k8gEJc" failed: Operation not permitted (1) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1668) [generator=3.1.3] pdsh@oleg229-client: oleg229-server: ssh exited with exit code 2