== sanity-flr test 200b: racing IO, mirror extend and resync ========================================================== 11:14:11 (1746112451) Starting client: oleg423-client.virtnet: -o user_xattr,flock 192.168.204.123@tcp:/lustre /mnt/lustre2 Starting client: oleg423-client.virtnet: -o user_xattr,flock 192.168.204.123@tcp:/lustre /mnt/lustre3 fail_loc=0x1423 resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..Extending file size to 4499520 .. Extending file size to 8644800 .. failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..Extending file size to 4517568 .. failed Extending file size to 4837600 .. lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed Extending file size to 5304128 .. Extending file size to 8914208 .. resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed Extending file size to 7539200 .. resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed Extending file size to 8420192 .. resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..Extending file size to 8567232 .. failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..done Extending file size to 8895200 .. resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..done resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..Extending file size to 9038720 .. done resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..done resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed Extending file size to 9222816 .. lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..done lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..done lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..done lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..done resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e resync_start' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed resync file /mnt/lustre3/f200b.sanity-flr with 'mirror_io resync -e delay_before_copy -d 1' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed lock to resync file /mnt/lustre3/f200b.sanity-flr with '/home/green/git/lustre-release/lustre/utils/lfs mirror resync' ..failed Waiting 3625 3626 3628 3630 3632 192.168.204.123@tcp:/lustre /mnt/lustre2 lustre rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose 0 0 Stopping client oleg423-client.virtnet /mnt/lustre2 (opts:) 192.168.204.123@tcp:/lustre /mnt/lustre3 lustre rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose 0 0 Stopping client oleg423-client.virtnet /mnt/lustre3 (opts:) lfs mirror mirror: fail to pread 0-9224192 of mirror 1: Invalid argument (22) lfs mirror mirror: error reading bytes 0-9224192 of mirror 1: Invalid argument (22) lfs mirror mirror: fail to mirror resync '/mnt/lustre/f200b.sanity-flr': Invalid argument (22) PID TTY STAT TIME COMMAND 1 ? Ss 0:04 /usr/lib/systemd/systemd --switched-root --system --deserialize 24 2 ? S 0:00 [kthreadd] 4 ? S< 0:00 [kworker/0:0H] 5 ? S 0:00 [kworker/u8:0] 6 ? S 0:00 [ksoftirqd/0] 7 ? S 0:00 [migration/0] 8 ? S 0:00 [rcu_bh] 9 ? S 0:03 [rcu_sched] 10 ? S 0:00 [rcuob/0] 11 ? S 0:03 [rcuos/0] 12 ? S< 0:00 [lru-add-drain] 13 ? S 0:00 [watchdog/0] 14 ? S 0:00 [watchdog/1] 15 ? S 0:00 [migration/1] 16 ? S 0:00 [ksoftirqd/1] 18 ? S< 0:00 [kworker/1:0H] 19 ? S 0:00 [rcuob/1] 20 ? S 0:03 [rcuos/1] 21 ? S 0:00 [watchdog/2] 22 ? S 0:00 [migration/2] 23 ? S 0:00 [ksoftirqd/2] 25 ? S< 0:00 [kworker/2:0H] 26 ? S 0:00 [rcuob/2] 27 ? S 0:02 [rcuos/2] 28 ? S 0:00 [watchdog/3] 29 ? S 0:00 [migration/3] 30 ? S 0:00 [ksoftirqd/3] 31 ? S 0:13 [kworker/3:0] 32 ? S< 0:00 [kworker/3:0H] 33 ? S 0:00 [rcuob/3] 34 ? S 0:03 [rcuos/3] 35 ? S 0:00 [kdevtmpfs] 36 ? S< 0:00 [netns] 37 ? S 0:00 [khungtaskd] 38 ? S< 0:00 [writeback] 39 ? S< 0:00 [kintegrityd] 40 ? S< 0:00 [bioset] 41 ? S< 0:00 [bioset] 42 ? S< 0:00 [bioset] 43 ? S< 0:00 [kblockd] 44 ? S< 0:00 [md] 45 ? S< 0:00 [watchdogd] 46 ? S 0:13 [kworker/2:1] 47 ? S< 0:00 [rpciod] 48 ? S< 0:00 [xprtiod] 49 ? S 0:00 [kworker/0:1] 50 ? S 0:00 [kworker/1:1] 52 ? S 0:00 [kswapd0] 53 ? SN 0:00 [ksmd] 54 ? SN 0:00 [khugepaged] 55 ? S< 0:00 [nfsiod] 56 ? S< 0:00 [crypto] 64 ? S< 0:00 [kthrotld] 66 ? S< 0:00 [kmpath_rdacd] 67 ? S< 0:00 [kpsmoused] 68 ? S 0:00 [kworker/0:2] 69 ? S< 0:00 [ipv6_addrconf] 82 ? S< 0:00 [deferwq] 279 ? S< 0:00 [ata_sff] 281 ? S 0:00 [scsi_eh_0] 282 ? S< 0:00 [scsi_tmf_0] 284 ? S 0:00 [scsi_eh_1] 285 ? S< 0:00 [scsi_tmf_1] 293 ? S 0:00 [kworker/u8:2] 447 ? Ss 0:00 @bd-client --systemd-mark 192.168.200.253 -N centos7 /dev/nbd0 -p -b4096 449 ? S< 0:00 [nbd0] 473 ? S 0:00 [jbd2/nbd0-8] 474 ? S< 0:00 [ext4-rsv-conver] 563 ? Ss 0:01 /usr/lib/systemd/systemd-journald 574 ? S 0:00 [kworker/2:2] 580 ? S 0:00 [kworker/1:2] 590 ? S 0:00 [kworker/3:2] 595 ? Ss 0:00 /usr/lib/systemd/systemd-udevd 715 ? S< 0:00 [kvm-irqfd-clean] 726 ? S< 0:00 [edac-poller] 735 ? S< 0:00 [kworker/0:1H] 767 ? S< 0:00 [kworker/3:1H] 768 ? S 0:00 [nfsv4.1-svc] 800 ? Ss 0:00 /usr/lib/systemd/systemd-logind 803 ? Ss 0:01 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation 806 ? Ssl 0:00 /usr/sbin/gssproxy -D 814 ? Ssl 0:00 /usr/sbin/NetworkManager --no-daemon 821 ? Ss 0:00 /usr/sbin/sshd -D 838 tty1 Ss+ 0:00 /sbin/agetty --noclear tty1 linux 839 ttyS0 Ss+ 0:00 /sbin/agetty --keep-baud 115200,38400,9600 ttyS0 vt220 840 ttyS1 Ss+ 0:00 /sbin/agetty --keep-baud 115200,38400,9600 ttyS1 vt220 853 ? Ssl 0:00 /usr/lib/polkit-1/polkitd --no-debug 857 ? S 0:00 /sbin/dhclient -d -q -sf /usr/libexec/nm-dhcp-helper -pf /var/run/dhclient-eth0.pid -lf /var/lib/NetworkManager/dhclient-f417ee32-20ae-4546-aabd-325b1eeefc2f-eth0.lease -cf /var/lib/NetworkManager/dhclient-eth0.conf eth0 888 ? Ssl 0:00 /usr/sbin/rsyslogd -n 990 ? S< 0:00 [kworker/2:1H] 1135 ? Ss 0:00 sshd: root@pts/0 1172 pts/0 Ss+ 0:00 /bin/bash /home/green/git/lustre-release/lustre/tests/auster -D /tmp/testlogs/ -r -k sanity-flr 1671 ? S< 0:00 [kworker/1:1H] 1950 ? S 0:00 [ptlrpc_hr00_000] 1951 ? S 0:00 [ptlrpc_hr01_000] 1952 ? S 0:00 [socknal_cd00] 1953 ? S 0:00 [socknal_cd01] 1954 ? S 0:00 [socknal_cd02] 1955 ? S 0:00 [socknal_cd03] 1956 ? S 0:00 [socknal_reaper] 1957 ? S 0:00 [socknal_sd00_00] 1958 ? S 0:00 [socknal_sd00_01] 1959 ? S 0:11 [socknal_sd01_00] 1960 ? S 0:11 [socknal_sd01_01] 1961 ? S 0:00 [acceptor_001] 1962 ? S 0:00 [monitor_thread] 1963 ? S 0:00 [lnet_discovery] 1964 ? S 0:01 [ptlrpcd_rcv] 1965 ? S 0:04 [ptlrpcd_00_00] 1966 ? S 0:04 [ptlrpcd_00_01] 1967 ? S 0:01 [ptlrpcd_01_00] 1968 ? S 0:01 [ptlrpcd_01_01] 3397 pts/0 S+ 0:00 bash /home/green/git/lustre-release/lustre/tests/sanity-flr.sh 3398 pts/0 S+ 0:00 tee -i /tmp/testlogs//sanity-flr.test_200b.test_log.oleg423-client.log 7015 pts/0 S+ 0:01 bash /home/green/git/lustre-release/lustre/tests/sanity-flr.sh 7513 ? S 0:00 /usr/lib/systemd/systemd-udevd 7534 pts/0 R+ 0:00 ps ax 7564 ? S 0:00 [ldlm_cb00_000] 7565 ? S 0:00 [ldlm_cb00_001] 7566 ? S 0:01 [ldlm_cb01_000] 7567 ? S 0:01 [ldlm_cb01_001] 7568 ? S 0:00 [ldlm_cn00_000] 7569 ? S 0:00 [ldlm_cn00_001] 7570 ? S 0:00 [ldlm_cn00_002] 7571 ? S 0:00 [ldlm_cn01_000] 7572 ? S 0:00 [ldlm_cn01_001] 7573 ? S 0:00 [ldlm_cn01_002] 7574 ? S 0:01 [ldlm_bl_01] 7575 ? S 0:01 [ldlm_bl_02] 7576 ? S 0:00 [ldlm_elt] 7577 ? S 0:00 [ll_cfg_requeue] 8275 ? S 0:01 [ldlm_cb01_002] 11690 ? S 0:01 [ldlm_bl_03] 14386 ? S 0:01 [ldlm_bl_04] 16360 ? S 0:00 [kworker/0:0] 16854 ? S 0:00 [kworker/2:0] 16919 ? S 0:00 [kworker/3:1] 17246 ? S 0:00 [kworker/1:0] 19511 ? S 0:00 [ldlm_cb00_002] 24119 ? S 0:01 [ldlm_bl_05] 31398 ? S 0:00 [ldlm_bl_06] 31399 ? S 0:00 [ldlm_bl_07] 31400 ? S 0:00 [ldlm_bl_08] 32423 ? S 0:00 [kworker/u8:4] sanity-flr test_200b: @@@@@@ FAIL: final resync failed Trace dump: = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7249:error() = /home/green/git/lustre-release/lustre/tests/sanity-flr.sh:3606:test_200b() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7622:run_one() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7685:run_one_logged() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7503:run_test() = /home/green/git/lustre-release/lustre/tests/sanity-flr.sh:3618:main() Dumping lctl log to /tmp/testlogs//sanity-flr.test_200b.*.1746112513.log rsync: chown "/tmp/testlogs/.sanity-flr.test_200b.debug_log.oleg423-server.1746112513.log.Wc7m7i" failed: Operation not permitted (1) rsync: chown "/tmp/testlogs/.sanity-flr.test_200b.dmesg.oleg423-server.1746112513.log.26pkSk" failed: Operation not permitted (1) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1651) [generator=3.1.2]