== sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 18:14:53 (1746137693) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.120907 s, 520 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=14369) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg148-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg148-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg148-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51282/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.201.148@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.201.148@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=14369) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg148-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg148-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg148-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51282/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.201.48,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.201.148@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.201.148@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 18008b6c-f775-436b-bd9d-1160d261f11d Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=18008b6c-f775-436b-bd9d-1160d261f11d (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.76049 s, 68.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0232539 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.013129 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0171871 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000298998 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00947 s, 51.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.578143 s, 90.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0226336 s, 0.0 kB/s dd: warning: partial read (135168 bytes); suggest iflag=fullblock 3+9 records in 3+9 records out 4194304 bytes (4.2 MB) copied, 0.0161685 s, 259 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.929993 s, 56.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.518369 s, 101 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831202 s, 631 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.194376 s, 43.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 dd: warning: partial read (425984 bytes); suggest iflag=fullblock 46+2 records in 46+2 records out 49283072 bytes (49 MB) copied, 0.813283 s, 60.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.45476 s, 36.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829141 s, 632 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.591666 s, 3.5 MB/s dd: warning: partial read (933888 bytes); suggest iflag=fullblock 0+2 records in 0+2 records out 1048576 bytes (1.0 MB) copied, 0.0264296 s, 39.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 dd: warning: partial read (557056 bytes); suggest iflag=fullblock 39+3 records in 39+3 records out 41943040 bytes (42 MB) copied, 0.488027 s, 85.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.13143 s, 46.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441256 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343796 s, 1.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0130639 s, 0.0 kB/s 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.0350992 s, 179 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 33+0 records in 33+0 records out 34603008 bytes (35 MB) copied, 0.228921 s, 151 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.083477 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.11975 s, 46.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 44+0 records in 44+0 records out 46137344 bytes (46 MB) copied, 0.458314 s, 101 MB/s dd: warning: partial read (176128 bytes); suggest iflag=fullblock 0+4 records in 0+4 records out 1048576 bytes (1.0 MB) copied, 0.0321498 s, 32.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 28+0 records in 28+0 records out 29360128 bytes (29 MB) copied, 0.483408 s, 60.7 MB/s dd: warning: partial read (4096 bytes); suggest iflag=fullblock 44+2 records in 44+2 records out 47185920 bytes (47 MB) copied, 0.0770335 s, 613 MB/s 45+0 records in 45+0 records out 47185920 bytes (47 MB) copied, 0.0532586 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.21035 s, 43.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0377265 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 12+0 records in 12+0 records out 12582912 bytes (13 MB) copied, 0.0879757 s, 143 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory dd: warning: partial read (778240 bytes); suggest iflag=fullblock dd: warning: partial read (344064 bytes); suggest iflag=fullblock 38+3 records in 38+3 records out 40894464 bytes (41 MB) copied, 0.270995 s, 151 MB/s 38+3 records in 38+3 records out 40894464 bytes (41 MB) copied, 0.125375 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.66932 s, 31.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.278179 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00641605 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000371172 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000335931 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00111088 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.3287 s, 39.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.721558 s, 72.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.066212 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00930822 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 28+0 records in 28+0 records out 29360128 bytes (29 MB) copied, 0.411343 s, 71.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.44509 s, 36.3 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104728 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578202 s, 907 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.446467 s, 75.2 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.398146 s, 2.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 42+0 records in 42+0 records out 44040192 bytes (44 MB) copied, 0.358794 s, 123 MB/s Detach /mnt/lustre/f100.sanity-pcc ... dd: warning: partial read (49152 bytes); suggest iflag=fullblock 45+5 records in 45+5 records out 48234496 bytes (48 MB) copied, 0.643591 s, 74.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.34826 s, 38.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.544456 s, 7.7 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0480409 s, 21.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 16+0 records in 16+0 records out 16777216 bytes (17 MB) copied, 0.255132 s, 65.8 MB/s dd: warning: partial read (561152 bytes); suggest iflag=fullblock 26+2 records in 26+2 records out 28311552 bytes (28 MB) copied, 0.0726939 s, 389 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.25591 s, 41.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.12251 s, 8.6 MB/s dd: warning: partial read (503808 bytes); suggest iflag=fullblock 0+2 records in 0+2 records out 1048576 bytes (1.0 MB) copied, 0.0383008 s, 27.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.549919 s, 76.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.17553 s, 44.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.733704 s, 71.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 26+0 records in 26+0 records out 27262976 bytes (27 MB) copied, 0.249676 s, 109 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00828041 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... dd: warning: partial read (90112 bytes); suggest iflag=fullblock 39+3 records in 39+3 records out 41943040 bytes (42 MB) copied, 0.283058 s, 148 MB/s 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.843092 s, 49.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.62483 s, 32.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.680324 s, 6.2 MB/s dd: warning: partial read (299008 bytes); suggest iflag=fullblock 0+2 records in 0+2 records out 1048576 bytes (1.0 MB) copied, 0.0343694 s, 30.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 18+0 records in 18+0 records out 18874368 bytes (19 MB) copied, 0.286411 s, 65.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 dd: warning: partial read (172032 bytes); suggest iflag=fullblock 29+12 records in 29+12 records out 32505856 bytes (33 MB) copied, 0.113269 s, 287 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.24676 s, 42.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.302505 s, 27.7 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0161725 s, 64.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.01511 s, 51.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.648843 s, 80.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... dd: warning: partial read (401408 bytes); suggest iflag=fullblock 5+3 records in 5+3 records out 6217728 bytes (6.2 MB) copied, 0.0488434 s, 127 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.308254 s, 105 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.0494196 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.954303 s, 54.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366471 s, 1.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.263805 s, 4.0 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0135179 s, 77.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.76387 s, 68.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.554497 s, 94.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770551 s, 680 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033793 s, 1.6 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0207035 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.0376 s, 50.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.613941 s, 85.4 MB/s 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.291808 s, 115 MB/s 38+0 records in 38+0 records out 39845888 bytes (40 MB) copied, 0.481243 s, 82.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.24406 s, 42.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513578 s, 1.0 GB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196352 s, 267 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.998351 s, 52.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.80009 s, 65.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484844 s, 1.1 GB/s dd: warning: partial read (978944 bytes); suggest iflag=fullblock 6+2 records in 6+2 records out 7340032 bytes (7.3 MB) copied, 0.274576 s, 26.7 MB/s 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.0163829 s, 512 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 39+0 records in 39+0 records out 40894464 bytes (41 MB) copied, 0.21337 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.35442 s, 38.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.12491 s, 46.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726513 s, 722 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0116797 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.897739 s, 58.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.767636 s, 68.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.651 s, 31.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 14+0 records in 14+0 records out 14680064 bytes (15 MB) copied, 0.376086 s, 39.0 MB/s dd: warning: partial read (557056 bytes); suggest iflag=fullblock 2+18 records in 2+18 records out 4194304 bytes (4.2 MB) copied, 0.0470578 s, 89.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 33+0 records in 33+0 records out 34603008 bytes (35 MB) copied, 0.324545 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.25612 s, 41.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.995654 s, 8.4 MB/s 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0550613 s, 38.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.337939 s, 105 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.0369104 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.04596 s, 50.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297361 s, 1.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.769087 s, 43.6 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.260083 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240461 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.702922 s, 74.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.24901 s, 42.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.816643 s, 5.1 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0181211 s, 57.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 26+0 records in 26+0 records out 27262976 bytes (27 MB) copied, 0.460052 s, 59.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.46672 s, 35.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148547 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657198 s, 798 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory dd: warning: partial read (987136 bytes); suggest iflag=fullblock 10+2 records in 10+2 records out 11534336 bytes (12 MB) copied, 0.240601 s, 47.9 MB/s 26+0 records in 26+0 records out 27262976 bytes (27 MB) copied, 0.0570407 s, 478 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.94197 s, 27.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.775986 s, 67.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113176 s, 463 MB/s 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.0629179 s, 133 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59b2/0x2000059f1:0x43:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.34288 s, 39.0 MB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.214364 s, 19.6 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0127715 s, 82.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.05812 s, 49.5 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now