-----============= acceptance-small: sanity-lnet ============----- Fri Oct 2 14:05:33 EDT 2026 MGS_OS_ID_LIKE=rhel centos fedora rocky MGS_OS_VERSION_ID=8.10 MGS_OS_ID=rocky MGS_OS_VERSION_CODE=134873088 MDS1_OS_VERSION_ID=8.10 MDS1_OS_VERSION_CODE=134873088 MDS1_OS_ID_LIKE=rhel centos fedora rocky MDS1_OS_ID=rocky OST1_OS_VERSION_CODE=134873088 OST1_OS_ID_LIKE=rhel centos fedora rocky OST1_OS_VERSION_ID=8.10 OST1_OS_ID=rocky CLIENT_OS_ID=rocky CLIENT_OS_VERSION_CODE=134873088 CLIENT_OS_VERSION_ID=8.10 CLIENT_OS_ID_LIKE=rhel centos fedora rocky Stopping clients: oleg129-client.virtnet /mnt/lustre (opts:) Stopping client oleg129-client.virtnet /mnt/lustre opts: Stopping clients: oleg129-client.virtnet /mnt/lustre2 (opts:) Stopping /mnt/lustre-mds1 (opts:-f) on oleg129-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg129-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg129-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg129-server unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: kmemleak disabled modules unloaded. Writer error: failed to resolve Netlink family id Stopping clients: oleg129-client.virtnet /mnt/lustre (opts:-f) Stopping clients: oleg129-client.virtnet /mnt/lustre2 (opts:-f) pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 LNET unconfigure error 22: Invalid argument unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: kmemleak disabled oleg129-server: LNET unconfigure error 22: Invalid argument modules unloaded. Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions ../libcfs/libcfs/libcfs options: 'cpu_npartitions=2' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure -a /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net show net: - net type: lo local NI(s): - nid: 0@lo status: up - net type: tcp local NI(s): - nid: 192.168.201.29@tcp status: up interfaces: 0: ens2 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens2: mtu 9000 qdisc fq_codel state UP group default qlen 1000 link/ether 56:44:47:3b:89:7d brd ff:ff:ff:ff:ff:ff altname enp0s2 inet 192.168.201.29/24 brd 192.168.201.255 scope global dynamic ens2 valid_lft 59675sec preferred_lft 59675sec inet6 fe80::5444:47ff:fe3b:897d/64 scope link valid_lft forever preferred_lft forever use SANITY_LNET_ONLY='' SANITY_LNET_EXCEPT='' oleg129-server: /home/green/git/lustre-release/lustre/tests/except/sanity-lnet.ex - see CLIENT_VERSION >= v2_16_51-17-g22ba71dccb8 (34683650 >= 34616081) for LU-18129, go 230 - see CLIENT_VERSION >= v2_16_57-110-ged70ccb471 (34683650 >= 34617710) for LU-15135, go 226 only running tests: 228 229 excepting tests: 253 254 410 Cleaning up LNet Writer error: failed to resolve Netlink family id unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: kmemleak disabled oleg129-server: LNET unconfigure error 22: Invalid argument modules unloaded. Total IP for ens2 found [1] Interface:IP are ens2:192.168.201.29 Using GW_NID:192.168.201.30@tcp repeat= (ONLY_REPEAT=) repeat_end= (ONLY_MINUTES=) == sanity-lnet test 228: Routes should stay up when health is decremented ========================================================== 14:07:21 (1790964441) SKIP: sanity-lnet test_228 Need at least 3 remote nodes found "oleg129-server" SKIP 228 (4s) repeat= (ONLY_REPEAT=) repeat_end= (ONLY_MINUTES=) == sanity-lnet test 229: Check NI status with lnetctl net disable/enable ========================================================== 14:07:25 (1790964445) Cleaning up LNet LNET unconfigure error 22: Invalid argument unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: kmemleak disabled oleg129-server: LNET unconfigure error 22: Invalid argument modules unloaded. ip link add 'test1pl' type veth peer name test1pg ip link set 'test1pl' up ip addr add "10.1.2.3/31" dev test1pg ip addr add "2001:db8:0:f101::1/64" dev test1pg ip link set test1pg up Waiting 10s for '0' Updated after 2s: want '0' got '0' Loading LNet and configuring DLC Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions ../libcfs/libcfs/libcfs options: 'cpu_npartitions=2' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if test1pg default via 192.168.201.254 dev ens2 192.168.201.0/24 dev ens2 proto kernel scope link src 192.168.201.29 NI 192.168.201.29@tcp expect status "up" found "up" NI 10.1.2.3@tcp expect status "up" found "up" NI 192.168.201.29@tcp expect manually_disabled "0" found "0" NI 10.1.2.3@tcp expect manually_disabled "0" found "0" Disable test1pg via lnetctl net disable /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --net tcp --if test1pg NI 192.168.201.29@tcp expect status "up" found "up" NI 10.1.2.3@tcp expect status "down" found "down" NI 192.168.201.29@tcp expect manually_disabled "0" found "0" NI 10.1.2.3@tcp expect manually_disabled "1" found "1" /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping 192.168.201.29@tcp ping: - primary nid: 192.168.201.29@tcp Multi-Rail: false peer_ni: - nid: 192.168.201.29@tcp - nid: 10.1.2.3@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping --source 10.1.2.3@tcp 192.168.201.29@tcp manage: - ping: errno: -100 descr: ! 'failed to ping 192.168.201.29@tcp: Network is down' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping 10.1.2.3@tcp manage: - ping: errno: -5 descr: ! 'failed to ping 10.1.2.3@tcp: Input/output error' Re-enable test1pg via lnetctl net enable /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net enable --net tcp --if test1pg NI 192.168.201.29@tcp expect status "up" found "up" NI 10.1.2.3@tcp expect status "up" found "up" NI 10.1.2.3@tcp expect manually_disabled "0" found "0" /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping --source 10.1.2.3@tcp 10.1.2.3@tcp ping: - primary nid: 10.1.2.3@tcp Multi-Rail: false peer_ni: - nid: 192.168.201.29@tcp - nid: 10.1.2.3@tcp Disable ens2 via lnetctl net disable /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --net tcp --if ens2 NI 192.168.201.29@tcp expect status "down" found "down" NI 10.1.2.3@tcp expect status "up" found "up" NI 192.168.201.29@tcp expect manually_disabled "1" found "1" NI 10.1.2.3@tcp expect manually_disabled "0" found "0" /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping 10.1.2.3@tcp ping: - primary nid: 10.1.2.3@tcp Multi-Rail: false peer_ni: - nid: 192.168.201.29@tcp - nid: 10.1.2.3@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping --source 192.168.201.29@tcp 10.1.2.3@tcp manage: - ping: errno: -100 descr: ! 'failed to ping 10.1.2.3@tcp: Network is down' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping 192.168.201.29@tcp manage: - ping: errno: -5 descr: ! 'failed to ping 192.168.201.29@tcp: Input/output error' Re-enable ens2 via lnetctl net enable /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net enable --net tcp --if ens2 NI 192.168.201.29@tcp expect status "up" found "up" NI 10.1.2.3@tcp expect status "up" found "up" Disable and enable both interfaces with a single command /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --net tcp --if ens2,test1pg NI 192.168.201.29@tcp expect status "down" found "down" NI 10.1.2.3@tcp expect status "down" found "down" /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl ping 192.168.201.29@tcp manage: - ping: errno: -113 descr: ! 'failed to ping 192.168.201.29@tcp: No route to host' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net enable --net tcp --if ens2,test1pg NI 192.168.201.29@tcp expect status "up" found "up" NI 10.1.2.3@tcp expect status "up" found "up" Verify lnetctl net disable rejects invalid usage /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --net tcp Missing required --net or --if parameter /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --if ens2 Missing required --net or --if parameter /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --net tcp --if ens2 --bogus net disable: disable network interface(s) --net: net name (e.g. tcp) --if: physical interface(s) (e.g. eth0 or eth0,eth1) /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --if ens2 --net net disable: disable network interface(s) --net: net name (e.g. tcp) --if: physical interface(s) (e.g. eth0 or eth0,eth1) /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --net tcp --if no_such_if --- set: - net: errno: -19 descr: ! "no matching LNet NI to enable / disable" ... /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net disable --net tcp999 --if ens2 --- set: - net: errno: -19 descr: ! "no matching LNet NI to enable / disable" ... NI 192.168.201.29@tcp expect status "up" found "up" NI 10.1.2.3@tcp expect status "up" found "up" Writer error: failed to resolve Netlink family id unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: kmemleak disabled oleg129-server: LNET unconfigure error 22: Invalid argument modules unloaded. pdsh@oleg129-client: oleg129-server: ssh exited with exit code 2 pdsh@oleg129-client: oleg129-client: ssh exited with exit code 2 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 2 pdsh@oleg129-client: oleg129-client: ssh exited with exit code 2 PASS 229 (400s) == sanity-lnet test complete, duration 512 sec =========== 14:14:06 (1790964846) Cleaning up LNet LNET unconfigure error 22: Invalid argument unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: kmemleak disabled oleg129-server: LNET unconfigure error 22: Invalid argument modules unloaded. Checking servers environments Checking clients oleg129-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions ../libcfs/libcfs/libcfs options: 'cpu_npartitions=2' ptlrpc/ptlrpc options: 'lbug_on_grant_miscount=1' quota/lquota options: 'hash_lqs_cur_bits=3' mdt/mdt options: 'mdt_enable_flr_ec=1' -rwxr-xr-x. 1 green green 1946968 Oct 2 13:55 /home/green/git/lustre-release/lustre/utils/mount.lustre -rwxr-xr-x. 1 root root 12 Jan 31 2019 /usr/sbin/mount.lustre mount --bind /home/green/git/lustre-release/lustre/utils/mount.lustre /usr/sbin/mount.lustre loading modules on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing load_modules_local oleg129-server: kmemleak disabled oleg129-server: Loading modules from /home/green/git/lustre-release/lustre oleg129-server: detected 4 online CPUs by sysfs oleg129-server: MODOPTS_LIBCFS= oleg129-server: Force libcfs to create 2 CPU partitions oleg129-server: ptlrpc/ptlrpc options: 'lbug_on_grant_miscount=1' oleg129-server: quota/lquota options: 'hash_lqs_cur_bits=3' oleg129-server: mdt/mdt options: 'mdt_enable_flr_ec=1' oleg129-server: /usr/sbin/ not writable for /usr/sbin/mount.lustre_tgt Setup mgs, mdt, osts pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg129-server: oleg129-server.virtnet: executing set_default_debug neterror net nettrace malloc all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-MDT0000 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start mds2: mount -t lustre -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg129-server: oleg129-server.virtnet: executing set_default_debug neterror net nettrace malloc all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-MDT0001 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start ost1: mount -t lustre -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg129-server: oleg129-server.virtnet: executing set_default_debug neterror net nettrace malloc all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-OST0000 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start ost2: mount -t lustre -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg129-server: oleg129-server.virtnet: executing set_default_debug neterror net nettrace malloc all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg129-client.virtnet: -o user_xattr,flock 192.168.201.129@tcp:/lustre /mnt/lustre Starting client oleg129-client.virtnet: -o user_xattr,flock 192.168.201.129@tcp:/lustre /mnt/lustre Started clients oleg129-client.virtnet: 192.168.201.129@tcp:/lustre on /mnt/lustre type lustre (rw,flock,checksum,encrypt,lazystatfs,lruresize,nolock,statfs_project,nosync_on_close,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff99cec5510000.idle_timeout=debug osc.lustre-OST0001-osc-ffff99cec5510000.idle_timeout=debug disable quota as required