== sanity-lnet test 211: Remote NI recovery checks ======= 04:40:57 (1731750057)
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if eth0
default via 192.168.204.254 dev eth0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if eth0
default via 192.168.204.254 dev eth0
default via 192.168.204.254 dev eth0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.204.43@tcp
discover:
- primary nid: 192.168.204.43@tcp
  Multi-Rail: true
  peer_ni:
  - nid: 192.168.204.43@tcp
  - nid: 192.168.204.43@tcp1
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.204.43@tcp --health 0
-p recovery queue should have 192.168.204.43@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.204.43@tcp
-p recovery queue should be empty
Waiting 20s for '0'
Waiting 10s for '0'
Updated after 15s: want '0' got '0'
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Check ping counts:
      -     nid: 192.168.204.43@tcp
                  ping_count: 0
      -     nid: 192.168.204.43@tcp1
                  ping_count: 0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.204.43@tcp --health 500
-p recovery queue should have 192.168.204.43@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.204.43@tcp
-p recovery queue should be empty
Waiting 20s for '0'
Updated after 7s: want '0' got '0'
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if eth0
default via 192.168.204.254 dev eth0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if eth0
default via 192.168.204.254 dev eth0
default via 192.168.204.254 dev eth0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.204.43@tcp
discover:
- primary nid: 192.168.204.43@tcp
  Multi-Rail: true
  peer_ni:
  - nid: 192.168.204.43@tcp
  - nid: 192.168.204.43@tcp1
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.204.43@tcp --health 0
Check ping counts:
Waiting 4s for '1'
      -     nid: 192.168.204.43@tcp
                  ping_count: 1
      -     nid: 192.168.204.43@tcp1
                  ping_count: 0
-p recovery queue should have 192.168.204.43@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.204.43@tcp
Check ping counts:
Waiting 14s for '4'
Updated after 10s: want '4' got '4'
      -     nid: 192.168.204.43@tcp
                  ping_count: 4
      -     nid: 192.168.204.43@tcp1
                  ping_count: 0
-p recovery queue should have 192.168.204.43@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
Peer NI recovery:
      nid-0: 192.168.204.43@tcp
/home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 900
pdsh@oleg443-client: oleg443-server: ssh exited with exit code 2
pdsh@oleg443-client: oleg443-server: ssh exited with exit code 2