== sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 01:19:26 (1780809566) 60+0 records in 60+0 records out 62914560 bytes (63 MB, 60 MiB) copied, 0.0764745 s, 823 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) bpf on /sys/fs/bpf type bpf (rw,nosuid,nodev,noexec,relatime,mode=700) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,rdma) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/misc type cgroup (rw,nosuid,nodev,noexec,relatime,misc) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) none on /sys/kernel/tracing type tracefs (rw,relatime) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,pagesize=2M) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) none on /mnt type ramfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /root/.ssh type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg333-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg333-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg333-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=378180k,mode=700) 192.168.200.253:/exports/testreports/64756/testresults/sanity-pcc-zfs-DNE-rocky8.10_x86_64-rocky8.10_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.133@tcp:/lustre on /mnt/lustre type lustre (rw,flock,checksum,encrypt,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.133@tcp:/lustre on /mnt/lustre2 type lustre (rw,flock,checksum,encrypt,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted. sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) bpf on /sys/fs/bpf type bpf (rw,nosuid,nodev,noexec,relatime,mode=700) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,rdma) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/misc type cgroup (rw,nosuid,nodev,noexec,relatime,misc) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) none on /sys/kernel/tracing type tracefs (rw,relatime) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,pagesize=2M) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) none on /mnt type ramfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /root/.ssh type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg333-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg333-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg333-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=378180k,mode=700) 192.168.200.253:/exports/testreports/64756/testresults/sanity-pcc-zfs-DNE-rocky8.10_x86_64-rocky8.10_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.33,local_lock=none,addr=192.168.200.253) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.133@tcp:/lustre on /mnt/lustre type lustre (rw,flock,checksum,encrypt,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.133@tcp:/lustre on /mnt/lustre2 type lustre (rw,flock,checksum,encrypt,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.47.3-wc2 (11-Nov-2025) Discarding device blocks: 8192/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: d7ee4052-0265-42a9-a889-e43e82908c6f Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d7ee4052-0265-42a9-a889-e43e82908c6f (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.362791 s, 145 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory dd: failed to open '/mnt/lustre/f100.sanity-pcc': Numerical result out of range sanity-pcc test_100: @@@@@@ FAIL: failed to write /mnt/lustre/f100.sanity-pcc 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.235476 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0204794 s, 2.6 GB/s Trace dump: = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7409:error() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:4344:test_100() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7785:run_one() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7848:run_one_logged() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7663:run_test() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:4399:main() Dumping lctl log to /tmp/testlogs//sanity-pcc.test_100.*.1780809568.log Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0221529 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0170428 s, 3.1 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0261231 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0194034 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0233156 s, 2.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0263209 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0196665 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.02145 s, 2.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0187057 s, 2.8 GB/s rsync: chown "/tmp/testlogs/.sanity-pcc.test_100.debug_log.oleg333-server.1780809568.log.Iib7VH" failed: Operation not permitted (1) rsync: chown "/tmp/testlogs/.sanity-pcc.test_100.dmesg.oleg333-server.1780809568.log.zs8YvE" failed: Operation not permitted (1) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1668) [generator=3.1.3] Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0235968 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0150867 s, 3.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0173194 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.013999 s, 3.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0182406 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0147915 s, 3.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0176288 s, 3.0 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0159189 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0183708 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.015529 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0219214 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0166668 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0145173 s, 3.6 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0220471 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0160569 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0209067 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0164956 s, 3.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0229317 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0156605 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0180209 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0193502 s, 2.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0167156 s, 3.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0349268 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0166446 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0204827 s, 2.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0179538 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0222842 s, 2.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0176961 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0222148 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0161961 s, 3.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0238165 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0166562 s, 3.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0174383 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0186528 s, 2.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0178803 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0191455 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.017817 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.018961 s, 2.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0159521 s, 3.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0194821 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0241133 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0164374 s, 3.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0217172 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0165973 s, 3.2 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.017324 s, 3.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0221529 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0172937 s, 3.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0197938 s, 2.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0176993 s, 3.0 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0181429 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0190095 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0163205 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0208412 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.01904 s, 2.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0181556 s, 2.9 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0190471 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0274123 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0278199 s, 1.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0206463 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0150254 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0191835 s, 2.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0175541 s, 3.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0150724 s, 3.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0221593 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0181155 s, 2.9 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0203642 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0152556 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0202306 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0211452 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0145639 s, 3.6 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0186252 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0268844 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0285512 s, 1.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0213146 s, 2.5 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0158942 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.019551 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0154628 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0177706 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0257381 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0260246 s, 2.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0329083 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0163836 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0191329 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.021973 s, 2.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0171973 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0184461 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0246324 s, 2.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0179384 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0206984 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0161025 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0206654 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0177886 s, 2.9 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0245441 s, 2.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0187966 s, 2.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.017712 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.015282 s, 3.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0193453 s, 2.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0174669 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0205576 s, 2.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0159543 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.018808 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0149125 s, 3.5 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0185902 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0152626 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0150426 s, 3.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.016114 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0207685 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0162935 s, 3.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0257782 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0175446 s, 3.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.021455 s, 2.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0251868 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0179785 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.020589 s, 2.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0167453 s, 3.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0195709 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0167061 s, 3.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0269493 s, 1.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0170294 s, 3.1 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.022412 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0168236 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0228004 s, 2.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0190365 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0181941 s, 2.9 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0181233 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0197246 s, 2.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0252197 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0264692 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0166009 s, 3.2 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0267941 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0168287 s, 3.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0251095 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0168382 s, 3.1 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0270349 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0184509 s, 2.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0213245 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0224895 s, 2.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0188783 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0269708 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0175177 s, 3.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0179232 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0173925 s, 3.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0184826 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0186016 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0212032 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0240747 s, 2.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0204652 s, 2.6 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0211174 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0182494 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0212605 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0195799 s, 2.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0313965 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0340963 s, 1.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0298664 s, 1.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0207389 s, 2.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0159362 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0175104 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0149643 s, 3.5 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0192888 s, 2.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0214009 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0142432 s, 3.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0191686 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0154709 s, 3.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0220391 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0143128 s, 3.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0245695 s, 2.1 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0139762 s, 3.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0152953 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0159049 s, 3.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0138606 s, 3.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0184175 s, 2.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0155858 s, 3.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0212685 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0153435 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0242335 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0222454 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0148682 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0201417 s, 2.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0178794 s, 2.9 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0156719 s, 3.3 GB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0176191 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0137875 s, 3.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0205801 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0220012 s, 2.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0137678 s, 3.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0237161 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0162839 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0147029 s, 3.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0207398 s, 2.5 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0306886 s, 1.7 GB/s52428800 bytes (52 MB, 50 MiB) copied, 0.0278373 s, 1.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0236349 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0164373 s, 3.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0179859 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0155422 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0181961 s, 2.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.013802 s, 3.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.019131 s, 2.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.017112 s, 3.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0145575 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.024442 s, 2.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0175116 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0153477 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0219534 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0141824 s, 3.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0200113 s, 2.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0141993 s, 3.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0222209 s, 2.4 GB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0134013 s, 3.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0224326 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0150885 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0246052 s, 2.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0139081 s, 3.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0141734 s, 3.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0205394 s, 2.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.023181 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0170085 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0154886 s, 3.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0166413 s, 3.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0146483 s, 3.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0150441 s, 3.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0204331 s, 2.6 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0210763 s, 2.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0186387 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0155173 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.017682 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0142324 s, 3.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0196137 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0151853 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0210648 s, 2.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0204261 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0136323 s, 3.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0167131 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0159408 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0192647 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0193422 s, 2.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0145423 s, 3.6 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0243271 s, 2.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0167317 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0151623 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0178317 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0154199 s, 3.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0263682 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0157098 s, 3.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0217315 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0299677 s, 1.7 GB/s52428800 bytes (52 MB, 50 MiB) copied, 0.0305505 s, 1.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0217883 s, 2.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0189403 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0192786 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0139944 s, 3.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/6145/0x2000061c1:0x84:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0203636 s, 2.6 GB/s sanity-pcc test_100: @@@@@@ FAIL: 1: write failed 50+0 records in 50+0 records out 52428800 bytes (52 MB, 50 MiB) copied, 0.0160827 s, 3.3 GB/s Trace dump: = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7409:error() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:4390:test_100() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7785:run_one() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7848:run_one_logged() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7663:run_test() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:4399:main() Dumping lctl log to /tmp/testlogs//sanity-pcc.test_100.*.1780809628.log rsync: chown "/tmp/testlogs/.sanity-pcc.test_100.debug_log.oleg333-server.1780809628.log.hLBC2R" failed: Operation not permitted (1) rsync: chown "/tmp/testlogs/.sanity-pcc.test_100.dmesg.oleg333-server.1780809628.log.WbkLW9" failed: Operation not permitted (1) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1668) [generator=3.1.3] Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now