== sanity-lnet test 210: Local NI recovery checks ======== 04:21:37 (1743495697)
Loading LNet and configuring DLC
/home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1001: echo: write error: Device or resource busy
/home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1002: echo: write error: Device or resource busy
Loading modules from /home/green/git/lustre-release/lustre
detected 4 online CPUs by sysfs
MODOPTS_LIBCFS=
Force libcfs to create 2 CPU partitions
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2
default via 192.168.206.254 dev ens2 192.168.206.0/24 dev ens2 proto kernel scope link src 192.168.206.32
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2
default via 192.168.206.254 dev ens2 192.168.206.0/24 dev ens2 proto kernel scope link src 192.168.206.32
default via 192.168.206.254 dev ens2 192.168.206.0/24 dev ens2 proto kernel scope link src 192.168.206.32
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.206.32@tcp
discover:
- primary nid: 192.168.206.32@tcp
  Multi-Rail: true
  peer_ni:
  - nid: 192.168.206.32@tcp
  - nid: 192.168.206.32@tcp1
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10
debug=+net
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 0 --nid 192.168.206.32@tcp
Check ping counts:
Waiting 10s for '2'
Updated after 4s: want '2' got '2'
                  ping_count: 0
      -     nid: 192.168.206.32@tcp
                  ping_count: 2
      -     nid: 192.168.206.32@tcp1
                  ping_count: 0
-l recovery queue should have 192.168.206.32@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
Local NI recovery:
      nid-0: 192.168.206.32@tcp
Check ping counts:
Waiting 10s for '3'
Updated after 2s: want '3' got '3'
                  ping_count: 0
      -     nid: 192.168.206.32@tcp
                  ping_count: 3
      -     nid: 192.168.206.32@tcp1
                  ping_count: 0
-l recovery queue should have 192.168.206.32@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
Local NI recovery:
      nid-0: 192.168.206.32@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2
default via 192.168.206.254 dev ens2 192.168.206.0/24 dev ens2 proto kernel scope link src 192.168.206.32
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2
default via 192.168.206.254 dev ens2 192.168.206.0/24 dev ens2 proto kernel scope link src 192.168.206.32
default via 192.168.206.254 dev ens2 192.168.206.0/24 dev ens2 proto kernel scope link src 192.168.206.32
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.206.32@tcp
discover:
- primary nid: 192.168.206.32@tcp
  Multi-Rail: true
  peer_ni:
  - nid: 192.168.206.32@tcp
  - nid: 192.168.206.32@tcp1
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4
debug=+net
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 0 --nid 192.168.206.32@tcp
Check ping counts:
Waiting 10s for '2'
Updated after 4s: want '2' got '2'
                  ping_count: 0
      -     nid: 192.168.206.32@tcp
                  ping_count: 2
      -     nid: 192.168.206.32@tcp1
                  ping_count: 0
-l recovery queue should have 192.168.206.32@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
Local NI recovery:
      nid-0: 192.168.206.32@tcp
Check ping counts:
Waiting 10s for '4'
Updated after 8s: want '4' got '4'
                  ping_count: 0
      -     nid: 192.168.206.32@tcp
                  ping_count: 4
      -     nid: 192.168.206.32@tcp1
                  ping_count: 0
-l recovery queue should have 192.168.206.32@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
Local NI recovery:
      nid-0: 192.168.206.32@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 900
pdsh@oleg632-client: oleg632-server: ssh exited with exit code 2
pdsh@oleg632-client: oleg632-server: ssh exited with exit code 2