-----============= acceptance-small: sanity-lnet ============----- Thu Sep 11 19:36:18 EDT 2025 mgs: Rocky Linux release 8.10 (Green Obsidian) MGS_OS_ID_LIKE=rhel centos fedora rocky MGS_OS_VERSION_ID=8.10 MGS_OS_ID=rocky MGS_OS_VERSION_CODE=134873088 mds1: Rocky Linux release 8.10 (Green Obsidian) MDS1_OS_VERSION_ID=8.10 MDS1_OS_VERSION_CODE=134873088 MDS1_OS_ID_LIKE=rhel centos fedora rocky MDS1_OS_ID=rocky ost1: Rocky Linux release 8.10 (Green Obsidian) OST1_OS_VERSION_CODE=134873088 OST1_OS_ID_LIKE=rhel centos fedora rocky OST1_OS_VERSION_ID=8.10 OST1_OS_ID=rocky client: Rocky Linux release 8.10 (Green Obsidian) CLIENT_OS_ID=rocky CLIENT_OS_VERSION_CODE=134873088 CLIENT_OS_VERSION_ID=8.10 CLIENT_OS_ID_LIKE=rhel centos fedora rocky Stopping clients: oleg129-client.virtnet /mnt/lustre (opts:) Stopping client oleg129-client.virtnet /mnt/lustre opts: Stopping clients: oleg129-client.virtnet /mnt/lustre2 (opts:) Stopping /mnt/lustre-mds1 (opts:-f) on oleg129-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg129-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg129-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg129-server /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy modules unloaded. Writer error: failed to resolve Netlink family id Stopping clients: oleg129-client.virtnet /mnt/lustre (opts:-f) Stopping clients: oleg129-client.virtnet /mnt/lustre2 (opts:-f) pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy LNET unconfigure error 22: Invalid argument /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy oleg129-server: LNET unconfigure error 22: Invalid argument oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy modules unloaded. /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1034: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1035: echo: write error: Device or resource busy Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions ../libcfs/libcfs/libcfs options: 'cpu_npartitions=2' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure -a /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net show net: - net type: lo local NI(s): - nid: 0@lo status: up - net type: tcp local NI(s): - nid: 192.168.201.29@tcp status: up interfaces: 0: ens2 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens2: mtu 9000 qdisc fq_codel state UP group default qlen 1000 link/ether 56:44:47:3b:89:7d brd ff:ff:ff:ff:ff:ff altname enp0s2 inet 192.168.201.29/24 brd 192.168.201.255 scope global dynamic ens2 valid_lft 59813sec preferred_lft 59813sec inet6 fe80::5444:47ff:fe3b:897d/64 scope link valid_lft forever preferred_lft forever oleg129-server: ls: cannot access '/home/green/git/lustre-release/lustre/tests/except/sanity-lnet.*ex': No such file or directory excepting tests: 253 254 410 Cleaning up LNet Writer error: failed to resolve Netlink family id /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy oleg129-server: LNET unconfigure error 22: Invalid argument oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy modules unloaded. Total IP for ens2 found [1] Interface:IP are ens2:192.168.201.29 Using GW_NID:192.168.201.30@tcp == sanity-lnet test 502: Verify lnetctl peer set --health (MR) ========================================================== 19:37:20 (1757633840) Loading LNet and configuring DLC /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1034: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1035: echo: write error: Device or resource busy Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure Cleaning up LNet Writer error: failed to resolve Netlink family id /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy oleg129-server: LNET unconfigure error 22: Invalid argument oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy modules unloaded. /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1034: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1035: echo: write error: Device or resource busy Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions ../libcfs/libcfs/libcfs options: 'cpu_npartitions=2' /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure -a oleg129-server: Writer error: failed to resolve Netlink family id oleg129-server: opening /dev/lnet failed: No such file or directory oleg129-server: hint: the kernel modules may not be loaded oleg129-server: IOC_LIBCFS_GET_NI error 2: No such file or directory pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 oleg129-server: oleg129-server.virtnet: executing load_lnet oleg129-server: Loading modules from /home/green/git/lustre-release/lustre oleg129-server: /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1034: echo: write error: Device or resource busy oleg129-server: /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1035: echo: write error: Device or resource busy oleg129-server: detected 4 online CPUs by sysfs oleg129-server: MODOPTS_LIBCFS= oleg129-server: Force libcfs to create 2 CPU partitions /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.201.129@tcp discover: - primary nid: 192.168.201.129@tcp Multi-Rail: true peer_ni: - nid: 192.168.201.129@tcp oleg129-server: oleg129-server.virtnet: executing lnet_if_list oleg129-server: oleg129-server.virtnet: executing /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 net: - net type: lo local NI(s): - nid: 0@lo status: up - net type: tcp local NI(s): - nid: 192.168.201.29@tcp status: up interfaces: 0: ens2 - net type: tcp1 local NI(s): - nid: 192.168.201.29@tcp1 status: up interfaces: 0: ens2 - primary nid: 192.168.201.129@tcp - nid: 192.168.201.129@tcp health stats: health value: 1000 - nid: 192.168.201.129@tcp1 health stats: health value: 1000 debug=+net debug=-1 /home/green/git/lustre-release/lustre/utils/lctl net_drop add -s *@tcp -d *@tcp -r 1 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 0 --nid 192.168.201.129@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 1000 --all /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 1000 --all Check that remote NI health has been changed hval: 2000 max_hval: 2000 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 0 --all /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 1000 --all /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 1000 --all Check that remote NI health has been changed hval: 2000 max_hval: 2000 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 500 --nid 192.168.201.129@tcp /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 1000 --all /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 1000 --all Check that remote NI health has been changed hval: 2000 max_hval: 2000 /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 500 --all /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --health 1000 --all /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 1000 --all Check that remote NI health has been changed hval: 2000 max_hval: 2000 oleg129-server: oleg129-server.virtnet: executing /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net del --net tcp1 --if ens2 Writer error: failed to resolve Netlink family id /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy modules unloaded. oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy oleg129-server: LNET unconfigure error 22: Invalid argument oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy pdsh@oleg129-client: oleg129-client: ssh exited with exit code 2 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 2 pdsh@oleg129-client: oleg129-client: ssh exited with exit code 2 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 2 PASS 502 (21s) == sanity-lnet test 510: add interface using altname and verify in LNet ========================================================== 19:37:41 (1757633861) Altname set: ens2 -> testalt0 Loading LNet and configuring DLC /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1034: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1035: echo: write error: Device or resource busy Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure --- add: - net: errno: -100 descr: ! "cannot add LNet NI" ... sanity-lnet test_510: @@@@@@ FAIL: failed to add testalt0 to LNet Trace dump: = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7370:error() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:5259:test_510() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7746:run_one() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7809:run_one_logged() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7609:run_test() = /home/green/git/lustre-release/lustre/tests/sanity-lnet.sh:5274:main() Dumping lctl log to /tmp/testlogs//sanity-lnet.test_510.*.1757633864.log oleg129-server: invalid parameter 'dump_kernel' oleg129-server: open(dump_kernel) failed: No such file or directory rsync: chown "/tmp/testlogs/.sanity-lnet.test_510.debug_log.oleg129-server.1757633864.log.0i7mp9" failed: Operation not permitted (1) rsync: chown "/tmp/testlogs/.sanity-lnet.test_510.dmesg.oleg129-server.1757633864.log.xYr6wN" failed: Operation not permitted (1) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1668) [generator=3.1.3] pdsh@oleg129-client: oleg129-server: ssh exited with exit code 2 FAIL 510 (4s) == sanity-lnet test complete, duration 87 sec ============ 19:37:45 (1757633865) sanity-lnet: FAIL: test_510 failed to add testalt0 to LNet Cleaning up LNet Writer error: failed to resolve Netlink family id /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy unloading modules via unload_modules_local on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing unload_modules_local oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 114: echo: write error: Device or resource busy oleg129-server: LNET unconfigure error 22: Invalid argument oleg129-server: /home/green/git/lustre-release/lustre/scripts/lustre_rmmod: line 156: echo: write error: Device or resource busy modules unloaded. Checking servers environments Checking clients oleg129-client.virtnet environments /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1034: echo: write error: Device or resource busy /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1035: echo: write error: Device or resource busy Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions ../libcfs/libcfs/libcfs options: 'cpu_npartitions=2' ptlrpc/ptlrpc options: 'lbug_on_grant_miscount=1' quota/lquota options: 'hash_lqs_cur_bits=3' ln: failed to create symbolic link '/sbin/.libs': Read-only file system loading modules on: 'oleg129-server' oleg129-server: oleg129-server.virtnet: executing load_modules_local oleg129-server: Loading modules from /home/green/git/lustre-release/lustre oleg129-server: /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1034: echo: write error: Device or resource busy oleg129-server: /home/green/git/lustre-release/lustre/tests/test-framework.sh: line 1035: echo: write error: Device or resource busy oleg129-server: detected 4 online CPUs by sysfs oleg129-server: MODOPTS_LIBCFS= oleg129-server: Force libcfs to create 2 CPU partitions oleg129-server: ptlrpc/ptlrpc options: 'lbug_on_grant_miscount=1' oleg129-server: quota/lquota options: 'hash_lqs_cur_bits=3' Setup mgs, mdt, osts pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start mds1: mount -t lustre -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg129-server: oleg129-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-MDT0000 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start mds2: mount -t lustre -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg129-server: oleg129-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-MDT0001 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start ost1: mount -t lustre -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg129-server: oleg129-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-OST0000 pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Start ost2: mount -t lustre -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg129-server: oleg129-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg129-client: oleg129-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg129-client.virtnet: -o user_xattr,flock 192.168.201.129@tcp:/lustre /mnt/lustre Starting client oleg129-client.virtnet: -o user_xattr,flock 192.168.201.129@tcp:/lustre /mnt/lustre Started clients oleg129-client.virtnet: 192.168.201.129@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,encrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff93da45f4e800.idle_timeout=debug osc.lustre-OST0001-osc-ffff93da45f4e800.idle_timeout=debug disable quota as required