== sanity-lnet test 211: Remote NI recovery checks ======= 16:30:49 (1786566649) /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 default via 192.168.201.254 dev ens2 192.168.201.0/24 dev ens2 proto kernel scope link src 192.168.201.5 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 default via 192.168.201.254 dev ens2 192.168.201.0/24 dev ens2 proto kernel scope link src 192.168.201.5 default via 192.168.201.254 dev ens2 192.168.201.0/24 dev ens2 proto kernel scope link src 192.168.201.5 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.201.5@tcp discover: - primary nid: 192.168.201.5@tcp Multi-Rail: true peer_ni: - nid: 192.168.201.5@tcp - nid: 192.168.201.5@tcp1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.201.5@tcp --health 0 -p recovery queue should have 192.168.201.5@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p Peer NI recovery: nid-0: 192.168.201.5@tcp -p recovery queue should be empty Waiting 20s for '0' Waiting 10s for '0' Updated after 14s: want '0' got '0' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p Check ping counts: - nid: 192.168.201.5@tcp ping_count: 0 - nid: 192.168.201.5@tcp1 ping_count: 0 /home/green/git/lustre-release/lustre/utils/lctl net_drop del -a /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.201.5@tcp --health 500 -p recovery queue should have 192.168.201.5@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p Peer NI recovery: nid-0: 192.168.201.5@tcp -p recovery queue should be empty Waiting 20s for '0' Updated after 6s: want '0' got '0' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 default via 192.168.201.254 dev ens2 192.168.201.0/24 dev ens2 proto kernel scope link src 192.168.201.5 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 default via 192.168.201.254 dev ens2 192.168.201.0/24 dev ens2 proto kernel scope link src 192.168.201.5 default via 192.168.201.254 dev ens2 192.168.201.0/24 dev ens2 proto kernel scope link src 192.168.201.5 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.201.5@tcp discover: - primary nid: 192.168.201.5@tcp Multi-Rail: true peer_ni: - nid: 192.168.201.5@tcp - nid: 192.168.201.5@tcp1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4 /home/green/git/lustre-release/lustre/utils/lctl net_drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/utils/lctl net_drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error /home/green/git/lustre-release/lustre/utils/lctl net_drop add -s *@tcp -d *@tcp -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.201.5@tcp --health 0 Check ping counts: Waiting 4s for '1' Update not seen after 4s: want '1' got '2' - nid: 192.168.201.5@tcp ping_count: 2 - nid: 192.168.201.5@tcp1 ping_count: 0 sanity-lnet test_211: @@@@@@ FAIL: Unexpected ping count Trace dump: = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7496:error() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:3498:check_ping_count() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:3657:test_211() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7873:run_one() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7936:run_one_logged() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7750:run_test() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:3672:main() Dumping lctl log to /tmp/testlogs//sanity-lnet.test_211.*.1786566684.log oleg105-server: invalid parameter 'dump_kernel' oleg105-server: open(dump_kernel) failed: No such file or directory rsync: chown "/tmp/testlogs/.sanity-lnet.test_211.debug_log.oleg105-server.1786566684.log.iGmvzz" failed: Operation not permitted (1) rsync: chown "/tmp/testlogs/.sanity-lnet.test_211.dmesg.oleg105-server.1786566684.log.28rL8S" failed: Operation not permitted (1) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1668) [generator=3.1.3] pdsh@oleg105-client: oleg105-server: ssh exited with exit code 2