== sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 15:33:20 (1746128000) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.102228 s, 615 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=28,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12919) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg459-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg459-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg459-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51271/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.159@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.159@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=28,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12919) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg459-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg459-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg459-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51271/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.59,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.159@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.159@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 5757c152-89e4-45e1-a75e-d8cd6c5e67e3 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5757c152-89e4-45e1-a75e-d8cd6c5e67e3 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.567526 s, 92.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.1743 s, 18.0 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0437913 s, 23.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.756607 s, 69.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.550014 s, 95.3 MB/s 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.262132 s, 196 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.12498 s, 46.6 MB/s 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.588986 s, 17.8 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0128448 s, 81.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.894815 s, 58.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.519222 s, 101 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0343841 s, 0.0 kB/s 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0174901 s, 420 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.742063 s, 70.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489957 s, 107 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523888 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347697 s, 1.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.231691 s, 104 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.20306 s, 43.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.903085 s, 58.1 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0628699 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00511 s, 52.2 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.657028 s, 79.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141228 s, 371 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00537 s, 52.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.92301 s, 56.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226091 s, 232 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00944037 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00952166 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.779943 s, 67.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.39218 s, 134 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0254491 s, 0.0 kB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0269404 s, 156 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.92643 s, 56.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.497033 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292632 s, 1.8 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0193672 s, 0.0 kB/s 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.0189559 s, 443 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.924076 s, 56.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.640397 s, 81.9 MB/s 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.0338847 s, 92.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.334251 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.918546 s, 57.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497028 s, 1.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.235692 s, 120 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0979349 s, 10.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.218101 s, 221 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.927124 s, 56.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287131 s, 1.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0226805 s, 46.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 33+0 records in 33+0 records out 34603008 bytes (35 MB) copied, 0.200672 s, 172 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.07581 s, 48.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425981 s, 1.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 9+0 records in 9+0 records out 9437184 bytes (9.4 MB) copied, 0.443455 s, 21.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0438497 s, 23.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 35+0 records in 35+0 records out 36700160 bytes (37 MB) copied, 0.0454599 s, 807 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.838558 s, 62.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.437914 s, 35.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0361129 s, 29.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.721833 s, 72.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.605598 s, 10.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0284662 s, 73.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.386133 s, 97.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.967211 s, 54.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.477169 s, 68.1 MB/s 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0346197 s, 60.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.857377 s, 61.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.547207 s, 95.8 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.026205 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.782883 s, 67.0 MB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.161867 s, 25.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0307534 s, 34.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.205682 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.812825 s, 64.5 MB/s 6+1 records in 6+1 records out 7188480 bytes (7.2 MB) copied, 0.0606033 s, 119 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.897237 s, 58.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 25+0 records in 25+0 records out 26214400 bytes (26 MB) copied, 0.618718 s, 42.4 MB/s 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0623988 s, 84.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.919183 s, 57.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.558249 s, 93.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408273 s, 1.3 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00875471 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.800462 s, 65.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.556292 s, 94.2 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0765716 s, 13.7 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0165182 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.639788 s, 81.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.527297 s, 99.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.055679 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0287307 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.09022 s, 48.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577669 s, 90.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 38+0 records in 38+0 records out 39845888 bytes (40 MB) copied, 0.186844 s, 213 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0511251 s, 20.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.06118 s, 49.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.729875 s, 71.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.487785 s, 107 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.465169 s, 51.8 MB/s 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.559733 s, 43.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.920799 s, 56.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.650865 s, 80.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.806206 s, 65.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.0884648 s, 273 MB/s 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.347153 s, 69.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.0414 s, 50.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.469894 s, 112 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.323819 s, 162 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.379643 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.924547 s, 56.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.65861 s, 79.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0201808 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00851849 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.155 s, 45.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.898047 s, 58.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0517533 s, 0.0 kB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0333656 s, 126 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.795967 s, 65.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359597 s, 146 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 12+0 records in 12+0 records out 12582912 bytes (13 MB) copied, 0.0830419 s, 152 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.194267 s, 173 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.814966 s, 64.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.594173 s, 88.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446674 s, 1.2 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0474052 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 25+0 records in 25+0 records out 26214400 bytes (26 MB) copied, 0.349387 s, 75.0 MB/s 26+0 records in 26+0 records out 27262976 bytes (27 MB) copied, 0.289523 s, 94.2 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.18433 s, 44.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.746014 s, 50.6 MB/s 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.0264221 s, 119 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.866554 s, 60.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0967939 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0423023 s, 0.0 kB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 35+0 records in 35+0 records out 36700160 bytes (37 MB) copied, 0.360197 s, 102 MB/s 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.130098 s, 387 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.12614 s, 46.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0541791 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00738859 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.387417 s, 125 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.978105 s, 53.6 MB/s 49+1 records in 49+1 records out 51453952 bytes (51 MB) copied, 0.155483 s, 331 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.15023 s, 41.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.213353 s, 167 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.16105 s, 45.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648593 s, 808 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780866 s, 671 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 35+0 records in 35+0 records out 36700160 bytes (37 MB) copied, 0.44901 s, 81.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b5/0x2000059f1:0x44:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.50653 s, 34.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.882842 s, 59.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560146 s, 936 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now umount: /mnt/pcc.d100.sanity-pcc: target is busy. (In some cases useful info about processes that use the device is found by lsof(8) or fuser(1))