== sanity-lnet test 211: Remote NI recovery checks ======= 18:16:48 (1746137808)
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if eth0
default via 192.168.201.254 dev eth0 192.168.201.0/24 dev eth0 proto kernel scope link src 192.168.201.52
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if eth0
default via 192.168.201.254 dev eth0 192.168.201.0/24 dev eth0 proto kernel scope link src 192.168.201.52
default via 192.168.201.254 dev eth0 192.168.201.0/24 dev eth0 proto kernel scope link src 192.168.201.52
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.201.52@tcp
discover:
- primary nid: 192.168.201.52@tcp
  Multi-Rail: true
  peer_ni:
  - nid: 192.168.201.52@tcp
  - nid: 192.168.201.52@tcp1
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.201.52@tcp --health 0
-p recovery queue should have 192.168.201.52@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.201.52@tcp
-p recovery queue should be empty
Waiting 20s for '0'
Waiting 10s for '0'
Updated after 16s: want '0' got '0'
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Check ping counts:
      -     nid: 192.168.201.52@tcp
                  ping_count: 0
      -     nid: 192.168.201.52@tcp1
                  ping_count: 0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.201.52@tcp --health 500
-p recovery queue should have 192.168.201.52@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.201.52@tcp
-p recovery queue should be empty
Waiting 20s for '0'
Updated after 7s: want '0' got '0'
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if eth0
default via 192.168.201.254 dev eth0 192.168.201.0/24 dev eth0 proto kernel scope link src 192.168.201.52
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if eth0
default via 192.168.201.254 dev eth0 192.168.201.0/24 dev eth0 proto kernel scope link src 192.168.201.52
default via 192.168.201.254 dev eth0 192.168.201.0/24 dev eth0 proto kernel scope link src 192.168.201.52
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.201.52@tcp
discover:
- primary nid: 192.168.201.52@tcp
  Multi-Rail: true
  peer_ni:
  - nid: 192.168.201.52@tcp
  - nid: 192.168.201.52@tcp1
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.201.52@tcp --health 0
Check ping counts:
Waiting 4s for '1'
      -     nid: 192.168.201.52@tcp
                  ping_count: 1
      -     nid: 192.168.201.52@tcp1
                  ping_count: 0
-p recovery queue should have 192.168.201.52@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.201.52@tcp
Check ping counts:
Waiting 14s for '4'
Waiting 4s for '4'
Updated after 11s: want '4' got '4'
      -     nid: 192.168.201.52@tcp
                  ping_count: 4
      -     nid: 192.168.201.52@tcp1
                  ping_count: 0
-p recovery queue should have 192.168.201.52@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.201.52@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 900
pdsh@oleg152-client: oleg152-server: ssh exited with exit code 2
pdsh@oleg152-client: oleg152-server: ssh exited with exit code 2