-----============= acceptance-small: sanity-pcc ============----- Thu May 1 23:55:56 EDT 2025 mgs: CentOS Linux release 7.9.2009 (Core) MGS_OS_VERSION_ID=7 MGS_OS_ID=centos MGS_OS_VERSION_CODE=117440512 MGS_OS_ID_LIKE=rhel fedora centos mds1: CentOS Linux release 7.9.2009 (Core) MDS1_OS_ID_LIKE=rhel fedora centos MDS1_OS_ID=centos MDS1_OS_VERSION_ID=7 MDS1_OS_VERSION_CODE=117440512 ost1: CentOS Linux release 7.9.2009 (Core) OST1_OS_VERSION_CODE=117440512 OST1_OS_VERSION_ID=7 OST1_OS_ID_LIKE=rhel fedora centos OST1_OS_ID=centos client: CentOS Linux release 7.9.2009 (Core) CLIENT_OS_ID=centos CLIENT_OS_ID_LIKE=rhel fedora centos CLIENT_OS_VERSION_ID=7 CLIENT_OS_VERSION_CODE=117440512 === sanity-pcc: start setup 23:56:06 (1746158166) === Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) oleg217-client.virtnet: executing check_config_client /mnt/lustre oleg217-client.virtnet: Checking config lustre mounted on /mnt/lustre Checking servers environments Checking clients oleg217-client.virtnet environments Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a993b000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b6e08800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a993b000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6e08800.idle_timeout=debug disable quota as required oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all osd-ldiskfs.track_declares_assert=1 === sanity-pcc: finish setup 23:56:27 (1746158187) === running as uid/gid/euid/egid 500/500/500/500, groups: 500 [true] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d0_runas_test/f7643] nobody:x:99: oleg217-server: ls: cannot access /home/green/git/lustre-release/lustre/tests/except/sanity-pcc.*ex: No such file or directory excepting tests: Killing existing copytools on oleg217-client.virtnet pkill: pidfile not valid Try `pkill --help' for more information. Set HSM on and start Set sanity-hsm HSM policy == sanity-pcc test 1a: Test manual lfs pcc attach with manual HSM restore ========================================================== 23:56:55 (1746158215) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992808 s, 528 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b2100f15-5d7f-4172-9339-bb45629bd298 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b2100f15-5d7f-4172-9339-bb45629bd298 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1a.sanity-pcc/d1a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0625799 s, 122 MB/s Restore testing... Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1a.sanity-pcc/f1a.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1a.sanity-pcc is idle now PASS 1a (13s) == sanity-pcc test 1b: Test manual lfs pcc attach with restore on remote access ========================================================== 23:57:08 (1746158228) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115405 s, 454 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a779e060-8b2e-4dbc-9464-bd8dc2b57ee9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a779e060-8b2e-4dbc-9464-bd8dc2b57ee9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1b.sanity-pcc/d1b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0557806 s, 137 MB/s Restore testing... Attach and detach testing Start to detach the /mnt/lustre/d1b.sanity-pcc/f1b.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1b.sanity-pcc is idle now PASS 1b (9s) == sanity-pcc test 1c: Test automated attach using Project ID with manual HSM restore ========================================================== 23:57:17 (1746158237) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a9939000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b6392000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a9939000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6392000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095978 s, 546 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e3ceb674-2f56-4000-bbdc-883ae929b2c9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e3ceb674-2f56-4000-bbdc-883ae929b2c9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1c.sanity-pcc/d1c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0482914 s, 159 MB/s Restore testing... Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1c.sanity-pcc/f1c.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1c.sanity-pcc is idle now PASS 1c (94s) == sanity-pcc test 1d: Test Project ID with remote access ========================================================== 23:58:51 (1746158331) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a7c04800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800ab2c8000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a7c04800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800ab2c8000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824363 s, 636 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f4d214bc-17a7-4820-bb6b-b5849e8248f4 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f4d214bc-17a7-4820-bb6b-b5849e8248f4 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1d.sanity-pcc/d1d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.042117 s, 182 MB/s Restore testing... Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1d.sanity-pcc/f1d.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1d.sanity-pcc is idle now PASS 1d (88s) == sanity-pcc test 1e: Test RW-PCC with non-root user ==== 00:00:19 (1746158419) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_1e/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_1e/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0110127 s, 93.0 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00335781 s, 305 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.006516 s, 157 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] touch: cannot touch '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre PASS 1e (7s) == sanity-pcc test 1f: Test auto RW-PCC cache with non-root user ========================================================== 00:00:26 (1746158426) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff88012bf4e000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012f5f0000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012bf4e000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012f5f0000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934593 s, 561 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 42718bd1-90ac-442a-8f49-13dd36248015 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=42718bd1-90ac-442a-8f49-13dd36248015 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00731984 s, 140 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00499711 s, 205 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=256] [count=1] 1+0 records in 1+0 records out 256 bytes (256 B) copied, 0.0039175 s, 65.3 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] touch: cannot touch '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1f.sanity-pcc is idle now PASS 1f (93s) == sanity-pcc test 1g: General permission test for RW-PCC ========================================================== 00:01:59 (1746158519) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115061 s, 456 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 03d29883-8d18-4e3e-949a-4e55d0ddbb3f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=03d29883-8d18-4e3e-949a-4e55d0ddbb3f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1g.sanity-pcc/d1g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00806792 s, 127 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00376651 s, 272 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] detach: cannot detach '/mnt/lustre/f1g.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00938118 s, 109 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1g.sanity-pcc is idle now PASS 1g (8s) == sanity-pcc test 2a: Test multi open when creating ===== 00:02:07 (1746158527) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800ab3f8000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012ab2a000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800ab3f8000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012ab2a000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110135 s, 476 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 292de4c3-27d0-49e9-83ec-f1dc6dc27912 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=292de4c3-27d0-49e9-83ec-f1dc6dc27912 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2a.sanity-pcc/multiop vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.26873 client oleg217-client.virtnet multiop_bg started multiop_pid=26883 ---------- 1 root root 14 May 2 00:03 /mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc/2340/0x240002341:0x1:0x0 Stopping multiop_pid=26883 (kill 26883 on oleg217-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d2a.sanity-pcc is idle now PASS 2a (96s) == sanity-pcc test 2b: Test multi remote open when creating ========================================================== 00:03:43 (1746158623) SKIP: sanity-pcc test_2b Need 2 or more clients, have 1 SKIP 2b (2s) == sanity-pcc test 2c: Test multi open on different mount points when creating ========================================================== 00:03:45 (1746158625) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b61df800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012f5f7800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b61df800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012f5f7800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112632 s, 465 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 44d7b310-c07b-49df-9c63-6378f6ab21e4 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=44d7b310-c07b-49df-9c63-6378f6ab21e4 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2c.sanity-pcc/d2c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2c.sanity-pcc/f2c.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.30407 client oleg217-client.virtnet multiop_bg started multiop_pid=30417 Stopping multiop_pid=30417 (kill 30417 on oleg217-client.virtnet) multiopen_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d2c.sanity-pcc is idle now PASS 2c (100s) == sanity-pcc test 3a: Repeat attach/detach operations === 00:05:25 (1746158725) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_3a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00911579 s, 112 kB/s Start to RW-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RW-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Start to RO-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RO-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Cleanup PCC backend on /mnt/lustre PASS 3a (7s) == sanity-pcc test 3b: Repeat attach/detach operations on multiple clients ========================================================== 00:05:32 (1746158732) SKIP: sanity-pcc test_3b Need 3 or more clients, have 1 SKIP 3b (2s) == sanity-pcc test 4: Auto cache test for mmap =========== 00:05:34 (1746158734) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b6e0b800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012a51c000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6e0b800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a51c000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723193 s, 725 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d4.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 452cb430-eb4f-41f8-b90a-7b2c65ae2278 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f4.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=452cb430-eb4f-41f8-b90a-7b2c65ae2278 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d4.sanity-pcc/d4.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' mmap test1: basic mmap operation (PASS, 0.018875s) mmap test2: MAP_PRIVATE not write back (PASS, 0.028463s) mmap test3: concurrent mmap ops on two nodes (PASS, 2.0343s) mmap test4: c1 write to f1 from mmapped f2, c2 write to f1 from mmapped f1 (PASS, 4.1211s) mmap test5: read/write file to/from the buffer which mmapped to just this file (PASS, 0.037719s) mmap test6: check mmap write/read content on two nodes (PASS, 0.67155s) mmap test7: file i/o with an unmapped buffer (SKIPPED, 0s) mmap test8: SIGBUS for beyond file size (SKIPPED, 0s) mmap test9: SIGBUS for negative file offset (SKIPPED, 0s) mmap test10: multi-thread mmap access (PASS, 0.091583s) mmap test11: multi-thread shared mmap (PASS, 0.13699s) mmap test10: mtime not change for readonly mmap access (PASS, 1.0218s) ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6e0b800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012a51c000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6e0b800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012a51c000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d4.sanity-pcc is idle now PASS 4 (104s) == sanity-pcc test 5: Mmap & cat a RW-PCC cached file ==== 00:07:18 (1746158838) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0928047 s, 565 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d5.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7f12d40d-6bf7-409f-93a4-a109bf890aaa Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f5.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7f12d40d-6bf7-409f-93a4-a109bf890aaa (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d5.sanity-pcc/d5.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d5.sanity-pcc is idle now PASS 5 (8s) == sanity-pcc test 6: Test mmap write on RW-PCC ========== 00:07:26 (1746158846) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0859924 s, 610 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d6.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 4bc03678-ae5b-472b-b579-2999add7fcb9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f6.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4bc03678-ae5b-472b-b579-2999add7fcb9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d6.sanity-pcc/d6.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d6.sanity-pcc is idle now PASS 6 (9s) == sanity-pcc test 7a: Fake file detached between fault() and page_mkwrite() for RW-PCC ========================================================== 00:07:35 (1746158855) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847588 s, 619 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a9406d8a-37e2-4117-a2f6-493ddbb0df13 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a9406d8a-37e2-4117-a2f6-493ddbb0df13 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7a.sanity-pcc/d7a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1412 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7a.sanity-pcc is idle now PASS 7a (7s) == sanity-pcc test 7b: Test the race with concurrent mkwrite and detach ========================================================== 00:07:42 (1746158862) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101573 s, 516 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: bbd52f3d-5d3d-410a-b481-46f9ab73d34e Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=bbd52f3d-5d3d-410a-b481-46f9ab73d34e (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7b.sanity-pcc/d7b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1413 fail_val=20 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7b.sanity-pcc is idle now PASS 7b (46s) == sanity-pcc test 8: Test fake -ENOSPC tolerance for RW-PCC ========================================================== 00:08:28 (1746158908) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105157 s, 499 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d8.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: fc8df572-92df-4a35-9344-3078cbd45b31 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f8.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=fc8df572-92df-4a35-9344-3078cbd45b31 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d8.sanity-pcc/d8.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d8.sanity-pcc is idle now PASS 8 (6s) == sanity-pcc test 9: Test -ENOSPC tolerance on loop PCC device for RW-PCC ========================================================== 00:08:35 (1746158915) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154838 s, 339 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.9a: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2523ee0a-c22f-43e3-a5f2-edae8e916efb Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f9.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2523ee0a-c22f-43e3-a5f2-edae8e916efb (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.9a/d9.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.9a/d9.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 1.32213 s, 47.6 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.9a is idle now PASS 9 (7s) == sanity-pcc test 10a: Test RW-PCC with user quota on loop PCC device ========================================================== 00:08:41 (1746158921) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811187 s, 646 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 360323cd-31a4-445b-92b3-76e2c59d9611 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=360323cd-31a4-445b-92b3-76e2c59d9611 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.200203 s, 78.6 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.184361 s, 85.3 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2400032e0:0x3:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10a.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10a.sanity-pcc/uquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 0.888899 s, 35.4 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10a.sanity-pcc is idle now PASS 10a (7s) == sanity-pcc test 10b: Test RW-PCC with group quota on loop PCC device ========================================================== 00:08:48 (1746158928) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765673 s, 685 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 08abac18-a058-40b2-b346-4c4c5e74921f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=08abac18-a058-40b2-b346-4c4c5e74921f (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.200447 s, 78.5 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.208141 s, 75.6 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x26:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10b.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10b.sanity-pcc/gquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 1.01584 s, 31.0 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10b.sanity-pcc is idle now PASS 10b (9s) == sanity-pcc test 10c: Test RO-PCC with user quota on loop PCC device ========================================================== 00:08:57 (1746158937) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745008 s, 704 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 65d3dc3a-071c-472b-a1d3-b5d881d6781b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=65d3dc3a-071c-472b-a1d3-b5d881d6781b (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.297527 s, 52.9 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.213891 s, 73.5 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10c.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10c.sanity-pcc is idle now PASS 10c (7s) == sanity-pcc test 10d: Test RO-PCC with group quota on loop PCC device ========================================================== 00:09:05 (1746158945) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822681 s, 637 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 942c5f91-2ac6-49c4-a6f1-d1690e85b607 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=942c5f91-2ac6-49c4-a6f1-d1690e85b607 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.272145 s, 57.8 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.245566 s, 64.1 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10d.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10d.sanity-pcc is idle now PASS 10d (7s) == sanity-pcc test 10e: Tolerate -EDQUOT failure when auto PCC-RO attach with user quota ========================================================== 00:09:12 (1746158952) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111204 s, 471 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 412bf140-0cab-414d-a41d-9206930d0d01 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=412bf140-0cab-414d-a41d-9206930d0d01 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10e.sanity-pcc/d10e.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: uid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0441018 s, 47.6 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.00973615 s, 215 MB/s /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x2c lmm_fid: [0x2000032e1:0x2c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xcc:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.149269 s, 35.1 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0132665 s, 395 MB/s file: /mnt/lustre/f10e.sanity-pcc, type: none /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x2c lmm_fid: [0x2000032e1:0x2c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xcc:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10e.sanity-pcc is idle now PASS 10e (8s) == sanity-pcc test 10f: Tolerate -EDQUOT failure when auto PCC-RO attach with group quota ========================================================== 00:09:20 (1746158960) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105567 s, 497 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 095de66b-a73d-4f45-b858-0c735272468b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=095de66b-a73d-4f45-b858-0c735272468b (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10f.sanity-pcc/d10f.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: gid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0268335 s, 78.2 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.00911361 s, 230 MB/s /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x2e lmm_fid: [0x2000032e1:0x2e:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xcd:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0882381 s, 59.4 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0120158 s, 436 MB/s file: /mnt/lustre/f10f.sanity-pcc, type: none /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x2e lmm_fid: [0x2000032e1:0x2e:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xcd:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10f.sanity-pcc is idle now PASS 10f (6s) == sanity-pcc test 11: Test attach fault injection with simulated PCC file path ========================================================== 00:09:27 (1746158967) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104425 s, 502 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d11.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3bef4b12-2952-4476-99fc-5472b77e9e20 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f11.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3bef4b12-2952-4476-99fc-5472b77e9e20 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d11.sanity-pcc/d11.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32d1 Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32d3 lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x32:0x0]: Permission denied (13) lfs pcc pcc: cannot attach with ID: 2: Permission denied (13) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Permission denied lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x33:0x0]: Invalid argument (22) lfs pcc pcc: cannot attach with ID: 2: Invalid argument (22) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Invalid argument Cleanup PCC backend on /mnt/lustre /mnt/pcc.d11.sanity-pcc is idle now PASS 11 (6s) == sanity-pcc test 12: RW-PCC attach races with concurrent HSM remove ========================================================== 00:09:33 (1746158973) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_12/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1414 fail_val=20 /mnt/lustre/f12.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre PASS 12 (25s) == sanity-pcc test 13a: Test auto RW-PCC create caching for UID/GID rule ========================================================== 00:09:59 (1746158999) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: uid={500} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0076892 s, 133 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0041193 s, 249 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00564479 s, 181 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: gid={500} lfs mkdir: dirstripe error on '/mnt/lustre/d13a.sanity-pcc': stripe already set lfs setdirstripe: cannot create dir '/mnt/lustre/d13a.sanity-pcc': File exists running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00362246 s, 283 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00784104 s, 131 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00733203 s, 140 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre PASS 13a (10s) == sanity-pcc test 13b: Test auto RW-PCC create caching for file name with wildcard ========================================================== 00:10:09 (1746159009) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13b/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13b/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: fname={*.h5 suffix.* Mid*dle} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00681552 s, 150 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/suffix.doc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00800109 s, 128 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/MidPADdle] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00863037 s, 119 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/Midpad] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00852332 s, 120 kB/s Cleanup PCC backend on /mnt/lustre PASS 13b (5s) == sanity-pcc test 13c: Check auto RW-PCC create caching for UID/GID/ProjID/fname rule ========================================================== 00:10:14 (1746159014) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b6393800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012f5f1000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6393800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012f5f1000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11659 s, 450 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d13c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 22d5b0e4-eba1-4f7f-883f-06f63f1ae0ce Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f13c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=22d5b0e4-eba1-4f7f-883f-06f63f1ae0ce (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100 200}&fname={*.h5},uid={500}&gid={500} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0275973 s, 37.1 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00345618 s, 296 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0099559 s, 103 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00397709 s, 257 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13c.sanity-pcc/ugidcache] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00973503 s, 105 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d13c.sanity-pcc is idle now PASS 13c (97s) == sanity-pcc test 14: Revocation of the layout lock should detach the file automatically ========================================================== 00:11:51 (1746159111) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_14/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear Cleanup PCC backend on /mnt/lustre PASS 14 (6s) == sanity-pcc test 15: Test auto attach at open when file is still valid cached ========================================================== 00:11:57 (1746159117) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102055 s, 514 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d15.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 4b586a82-54dd-4b1d-9d1e-605af649d9c2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f15.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4b586a82-54dd-4b1d-9d1e-605af649d9c2 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d15.sanity-pcc/d15.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach for non-root user running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0161243 s, 63.5 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] file: /mnt/lustre/d15.sanity-pcc/f15.sanity-pcc, type: readwrite, PCC_file: /d15.sanity-pcc/3abb/0x200003ab1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] Verify auto attach at open for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear Verify auto attach at open for RO-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d15.sanity-pcc is idle now PASS 15 (10s) == sanity-pcc test 16: Test detach with different options ========================================================== 00:12:07 (1746159127) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794282 s, 660 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d16.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 631c5b58-855e-42e8-9615-8e6900bc4403 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f16.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=631c5b58-855e-42e8-9615-8e6900bc4403 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d16.sanity-pcc/d16.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Test detach for RW-PCC Test for reusing valid PCC cache Test for the default detach Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Test detach for RO-PCC detach_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d16.sanity-pcc is idle now PASS 16 (9s) == sanity-pcc test 17: Test auto attach for layout refresh ========================================================== 00:12:16 (1746159136) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814594 s, 644 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d17.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 25619abb-a17d-4211-a46f-efb94c0e68ea Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f17.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=25619abb-a17d-4211-a46f-efb94c0e68ea (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d17.sanity-pcc/d17.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d17.sanity-pcc/d17.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 94 autocache: projid={100} Verify auto attach during IO for truncate Verify auto attach during IO for read/write multiop /mnt/lustre/f17.sanity-pcc vO_r TMPPIPE=/tmp/multiop_open_wait_pipe.22742 client oleg217-client.virtnet multiop_bg started multiop_pid=22752 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear Stopping multiop_pid=22752 (kill 22752 on oleg217-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d17.sanity-pcc is idle now PASS 17 (15s) == sanity-pcc test 18: Verify size correctness after re-attach the file ========================================================== 00:12:32 (1746159152) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102776 s, 510 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d18.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 34674d58-890e-4d2d-8b3e-67974c494ac6 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f18.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=34674d58-890e-4d2d-8b3e-67974c494ac6 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d18.sanity-pcc/d18.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d18.sanity-pcc/d18.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0999585 s, 42.0 MB/s file: /mnt/lustre/f18.sanity-pcc, type: readwrite, PCC_file: /d18.sanity-pcc/3aa7/0x200003ab1:0x16:0x0, open_count: 0, flags: 0 file: /mnt/lustre/f18.sanity-pcc, type: none /mnt/lustre/f18.sanity-pcc has size 4194304 OK 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00650058 s, 158 kB/s /mnt/lustre2/f18.sanity-pcc has size 1049600 OK file: /mnt/lustre/f18.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d18.sanity-pcc is idle now PASS 18 (8s) == sanity-pcc test 19: Verify the file re-attach works as expected ========================================================== 00:12:39 (1746159159) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100531 s, 522 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d19.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9efbe74a-4c40-447e-b015-347307b5679b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f19.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9efbe74a-4c40-447e-b015-347307b5679b (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d19.sanity-pcc/d19.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d19.sanity-pcc is idle now PASS 19 (6s) == sanity-pcc test 20: Auto attach works after the inode was once evicted from cache ========================================================== 00:12:45 (1746159165) 120+0 records in 120+0 records out 125829120 bytes (126 MB) copied, 0.234522 s, 537 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d20.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/122880 done Creating filesystem with 122880 1k blocks and 30720 inodes Filesystem UUID: 23dd46e5-7cc7-4078-b16a-355f4afcb893 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/15 done Writing inode tables: 0/15 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/15 done /tmp/f20.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=23dd46e5-7cc7-4078-b16a-355f4afcb893 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d20.sanity-pcc/d20.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d20.sanity-pcc is idle now PASS 20 (7s) == sanity-pcc test 21a: PCC-RO storing as a plain HSM mirror component for plain layout ========================================================== 00:12:52 (1746159172) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119859 s, 437 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8fc72c95-3590-4001-89df-77620bcd3af3 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8fc72c95-3590-4001-89df-77620bcd3af3 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21a.sanity-pcc/d21a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Plain layout info before PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x22 lmm_fid: [0x200003ab1:0x22:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 234 0xea 0x280000401 FLR layout info after PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x22 lmm_fid: [0x200003ab1:0x22:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x22 lmm_fid: [0x200003ab1:0x22:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC attach /mnt/lustre/f21a.sanity-pcc again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x22 lmm_fid: [0x200003ab1:0x22:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC detach '/mnt/lustre/f21a.sanity-pcc' again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x22 lmm_fid: [0x200003ab1:0x22:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21a.sanity-pcc is idle now PASS 21a (4s) == sanity-pcc test 21b: PCC-RO stroing as a plain HSM mirror component for FLR layouts ========================================================== 00:12:56 (1746159176) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919056 s, 570 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 0b9772ce-0e7f-476d-beb0-a4efb2d00828 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0b9772ce-0e7f-476d-beb0-a4efb2d00828 (extents) (64bit) (large files) (huge files) FLR layout before PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xeb:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xeb:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xec:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xeb:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xeb:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xec:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xeb:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xeb:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xec:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc' again: /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xeb:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xeb:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xec:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xeb:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xea:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x23 lmm_fid: [0x200003ab1:0x23:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xeb:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xec:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21b.sanity-pcc is idle now PASS 21b (5s) == sanity-pcc test 21c: Verify HSM release works storing PCC-RO as HSM mirror component ========================================================== 00:13:01 (1746159181) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825719 s, 635 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: dec02cf8-b445-4214-8728-4dfe352d0e30 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=dec02cf8-b445-4214-8728-4dfe352d0e30 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21c.sanity-pcc/d21c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' /mnt/lustre/f21c.sanity-pcc: (0x00000009) exists archived, archive_id:2 /mnt/lustre/f21c.sanity-pcc lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 0 lmm_stripe_offset: 0 lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' /mnt/lustre/f21c.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21c.sanity-pcc is idle now PASS 21c (6s) == sanity-pcc test 21d: Write should invalidate PCC-RO caching ========================================================== 00:13:07 (1746159187) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941586 s, 557 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 382932f1-7394-4ab9-b1ad-19649aa6b9bf Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=382932f1-7394-4ab9-b1ad-19649aa6b9bf (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21d.sanity-pcc/d21d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21d.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 236 0xec 0x2c0000401 PCC-RO attach '/mnt/lustre/f21d.sanity-pcc': /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x26 lmm_fid: [0x200003ab1:0x26:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xec:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Write invalidated PCC-RO cache: /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x26 lmm_fid: [0x200003ab1:0x26:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xec:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21d.sanity-pcc is idle now PASS 21d (7s) == sanity-pcc test 21e: Truncate should invalidate PCC-RO caching ========================================================== 00:13:14 (1746159194) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893178 s, 587 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 75e8616b-a62e-4c28-a4a7-0ad526055496 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=75e8616b-a62e-4c28-a4a7-0ad526055496 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21e.sanity-pcc/d21e.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21e.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 238 0xee 0x280000401 PCC-RO attach '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x28 lmm_fid: [0x200003ab1:0x28:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xee:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Trucate invalidate PCC-RO file '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x28 lmm_fid: [0x200003ab1:0x28:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xee:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21e.sanity-pcc is idle now PASS 21e (5s) == sanity-pcc test 21f: mmap write should invalidate PCC-RO caching ========================================================== 00:13:19 (1746159199) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103462 s, 507 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1efe7cee-9ac4-467b-9051-4287e433e18d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1efe7cee-9ac4-467b-9051-4287e433e18d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21f.sanity-pcc/d21f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21f.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 237 0xed 0x2c0000401 PCC-RO attach '/mnt/lustre/f21f.sanity-pcc': /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xed:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Mmap write invalidate PCC-RO caching: /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xed:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21f.sanity-pcc is idle now PASS 21f (6s) == sanity-pcc test 21g: PCC-RO for file under FLR write pending state ========================================================== 00:13:25 (1746159205) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139623 s, 376 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b2ccb3a1-618a-492d-bd6a-253dbeaef682 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b2ccb3a1-618a-492d-bd6a-253dbeaef682 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21g.sanity-pcc/d21g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: wp lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2b lmm_fid: [0x200003ab1:0x2b:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xef:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xee:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2b lmm_fid: [0x200003ab1:0x2b:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xef:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xf0:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp,pccro lcm_layout_gen: 4 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2b lmm_fid: [0x200003ab1:0x2b:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xef:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xee:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2b lmm_fid: [0x200003ab1:0x2b:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xef:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xf0:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after Write invalidate '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp lcm_layout_gen: 5 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2b lmm_fid: [0x200003ab1:0x2b:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xef:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xee:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2b lmm_fid: [0x200003ab1:0x2b:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xef:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xf0:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21g.sanity-pcc is idle now PASS 21g (5s) == sanity-pcc test 21h: Extend mirror once file was PCC-RO cached ========================================================== 00:13:30 (1746159210) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113569 s, 462 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21h.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ddda5f56-90cb-4d10-b30b-51b5292f56da Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21h.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ddda5f56-90cb-4d10-b30b-51b5292f56da (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21h.sanity-pcc/d21h.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf0:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xf1:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf2:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf1:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf0:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xf1:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf2:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf1:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after extend a mirror: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf0:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xf1:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf2:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf1:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf3:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf2:0x0] } file: /mnt/lustre/f21h.sanity-pcc, type: none FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc' again: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf0:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x280000401:0xf1:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf2:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf1:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2e lmm_fid: [0x200003ab1:0x2e:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf3:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x2c0000401:0xf2:0x0] } Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21h.sanity-pcc is idle now PASS 21h (6s) == sanity-pcc test 21i: HSM release increase layout gen, should invalidate PCC-RO cache ========================================================== 00:13:36 (1746159216) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160612 s, 326 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21i.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: fde0c31f-ee68-4d01-a4d8-8a67eb43930e Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21i.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=fde0c31f-ee68-4d01-a4d8-8a67eb43930e (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21i.sanity-pcc/d21i.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21i.sanity-pcc: (0x00000009) exists archived, archive_id:2 Layout after HSM release /mnt/lustre/f21i.sanity-pcc: /mnt/lustre/f21i.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 16 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC state /mnt/lustre/f21i.sanity-pcc: file: /mnt/lustre/f21i.sanity-pcc, type: none Layout after PCC-RO attach /mnt/lustre/f21i.sanity-pcc again: /mnt/lustre/f21i.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 0 obdidx objid objid group 0 245 0xf5 0x280000401 PCC state: file: /mnt/lustre/f21i.sanity-pcc, type: readonly, PCC_file: /d21i.sanity-pcc/3a81/0x200003ab1:0x30:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21i.sanity-pcc is idle now PASS 21i (9s) == sanity-pcc test 21j: PCC-RO for encrypted file ======== 00:13:45 (1746159225) SKIP: sanity-pcc test_21j client encryption not supported SKIP 21j (2s) == sanity-pcc test 22: Test RO-PCC attach for the HSM released file ========================================================== 00:13:47 (1746159227) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132356 s, 396 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d22.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: bd86ac23-cff1-41c4-8716-54311927e72f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f22.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=bd86ac23-cff1-41c4-8716-54311927e72f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d22.sanity-pcc/d22.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 2 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf6:0x0] } - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after HSM release /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 16 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc again: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 245 0xf5 0x2c0000401 Layout after PCC-RO detach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 245 0xf5 0x2c0000401 Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d22.sanity-pcc is idle now PASS 22 (11s) == sanity-pcc test 23: Test write/truncate/mmap-write invalidating RO-PCC caching ========================================================== 00:13:58 (1746159238) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108001 s, 485 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d23.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9b867753-0526-431e-8890-50704841371d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f23.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9b867753-0526-431e-8890-50704841371d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d23.sanity-pcc/d23.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Write should invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3d lmm_fid: [0x200003ab1:0x3d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf8:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3d lmm_fid: [0x200003ab1:0x3d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf8:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Truncate invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3d lmm_fid: [0x200003ab1:0x3d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf8:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 8594128896 lhm_archive_ver: 10737419265 lhm_archive_uuid: '' Finish trucate operation Mmap write invalidates RO-PCC caching /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3d lmm_fid: [0x200003ab1:0x3d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf8:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 8594128896 lhm_archive_ver: 10737419265 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 7 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3d lmm_fid: [0x200003ab1:0x3d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf8:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 8594128896 lhm_archive_ver: 10737419265 lhm_archive_uuid: '' Mmap write /mnt/lustre/f23.sanity-pcc via multiop /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 8 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3d lmm_fid: [0x200003ab1:0x3d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0xf8:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d23.sanity-pcc is idle now PASS 23 (7s) == sanity-pcc test 24a: Test RO-PCC with non-root user === 00:14:05 (1746159245) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121704 s, 431 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 4c1b61f0-f7b5-40d0-9649-ae70c1bb5546 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4c1b61f0-f7b5-40d0-9649-ae70c1bb5546 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: a0 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00657581 s, 156 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00273177 s, 375 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af1/0x200003ab1:0x40:0x0] touch: cannot touch '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af1/0x200003ab1:0x40:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af1/0x200003ab1:0x40:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af1/0x200003ab1:0x40:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af1/0x200003ab1:0x40:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af1/0x200003ab1:0x40:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24a.sanity-pcc is idle now PASS 24a (6s) == sanity-pcc test 24b: General permission test for RO-PCC ========================================================== 00:14:11 (1746159251) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900372 s, 582 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5f4d5cbb-493e-40ca-bc5c-dc098af23061 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5f4d5cbb-493e-40ca-bc5c-dc098af23061 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24b.sanity-pcc/d24b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00443026 s, 231 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00234167 s, 437 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] detach: cannot detach '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00439171 s, 233 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24b.sanity-pcc is idle now PASS 24b (6s) == sanity-pcc test 25: Tolerate fake read failure for RO-PCC ========================================================== 00:14:17 (1746159257) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113235 s, 463 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d25.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 38b8491d-16e0-43b2-99f9-7bf9dc75bed2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f25.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=38b8491d-16e0-43b2-99f9-7bf9dc75bed2 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d25.sanity-pcc/d25.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 fail_loc=0 fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d25.sanity-pcc is idle now PASS 25 (7s) == sanity-pcc test 26: Repeat the attach/detach when the file has multiple openers ========================================================== 00:14:24 (1746159264) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070236 s, 746 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d26.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: cda610b7-b647-4bdb-a6dc-05f829afbec7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f26.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=cda610b7-b647-4bdb-a6dc-05f829afbec7 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d26.sanity-pcc/d26.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.6177 client oleg217-client.virtnet multiop_bg started multiop_pid=6187 Stopping multiop_pid=6187 (kill 6187 on oleg217-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.6292 client oleg217-client.virtnet multiop_bg started multiop_pid=6302 Stopping multiop_pid=6302 (kill 6302 on oleg217-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.6407 client oleg217-client.virtnet multiop_bg started multiop_pid=6417 Stopping multiop_pid=6417 (kill 6417 on oleg217-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.6522 client oleg217-client.virtnet multiop_bg started multiop_pid=6532 Waiting 200s for 'SUCCEED' Stopping multiop_pid=6532 (kill 6532 on oleg217-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.6672 client oleg217-client.virtnet multiop_bg started multiop_pid=6682 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=6682 (kill 6682 on oleg217-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d26.sanity-pcc is idle now PASS 26 (30s) == sanity-pcc test 27: Auto attach at open when the file has multiple openers ========================================================== 00:14:54 (1746159294) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111962 s, 468 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d27.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 836a3e70-f815-4cf5-9743-881bc6f1e9fb Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f27.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=836a3e70-f815-4cf5-9743-881bc6f1e9fb (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d27.sanity-pcc/d27.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7599 client oleg217-client.virtnet multiop_bg started multiop_pid=7609 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=7609 (kill 7609 on oleg217-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7744 client oleg217-client.virtnet multiop_bg started multiop_pid=7754 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=7754 (kill 7754 on oleg217-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7888 client oleg217-client.virtnet multiop_bg started multiop_pid=7898 Stopping multiop_pid=7898 (kill 7898 on oleg217-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.8007 client oleg217-client.virtnet multiop_bg started multiop_pid=8017 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012f5f1000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b6393800.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012f5f1000.lru_size=clear Stopping multiop_pid=8017 (kill 8017 on oleg217-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d27.sanity-pcc is idle now PASS 27 (28s) == sanity-pcc test 28: RW-PCC attach should fail when the file has cluster-wide openers ========================================================== 00:15:22 (1746159322) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102672 s, 511 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d28.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7e1621c3-6553-4cc8-94f7-d75fc1330c9c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f28.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7e1621c3-6553-4cc8-94f7-d75fc1330c9c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d28.sanity-pcc/d28.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.8884 client oleg217-client.virtnet multiop_bg started multiop_pid=8894 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x56:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Stopping multiop_pid=8894 (kill 8894 on oleg217-client.virtnet) multiop /mnt/lustre2/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7643 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x56:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Cleanup PCC backend on /mnt/lustre /mnt/pcc.d28.sanity-pcc is idle now PASS 28 (10s) == sanity-pcc test 29a: Auto readonly caching on RO-PCC backend for O_RDONLY open ========================================================== 00:15:32 (1746159332) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff88012ab2a800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012d01b800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012ab2a800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012d01b800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689853 s, 760 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 07a42265-c63e-467d-b1fd-488c86824291 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=07a42265-c63e-467d-b1fd-488c86824291 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 100 P /mnt/lustre/d29a.sanity-pcc/f29a.sanity-pcc Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29a.sanity-pcc is idle now PASS 29a (94s) == sanity-pcc test 29b: Auto PCC-RO attach in atomic_open ========================================================== 00:17:06 (1746159426) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134741 s, 389 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 87e09e2a-923f-4805-8f21-3a3cf0df0ace Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=87e09e2a-923f-4805-8f21-3a3cf0df0ace (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d29b.sanity-pcc/d29b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.dat} 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.0107814 s, 380 kB/s 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.0110798 s, 370 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.014347 s, 285 kB/s vm.drop_caches = 3 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.0126257 s, 324 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29b.sanity-pcc is idle now PASS 29b (8s) == sanity-pcc test 30: Test lctl pcc del command ========= 00:17:14 (1746159434) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906726 s, 578 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d30.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 22988759-b9b7-4efb-9818-6e9212141695 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f30.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=22988759-b9b7-4efb-9818-6e9212141695 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 3e autocache: projid={100} check file 0:'4284' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'4286' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'4282' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 1:'0x240004280:0x4:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4284' Handle the file: 0x240004280:0x4:0x0 Remove non-cached file: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4284/0x240004280:0x4:0x0 flags: 1 check file 1:'0x240004280:0x6:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4286' Handle the file: 0x240004280:0x6:0x0 Detach and remove the PCC cached file: 0x240004280:0x6:0x0 check file 1:'0x240004280:0x2:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4282' Handle the file: 0x240004280:0x2:0x0 Detach and remove the PCC cached file: 0x240004280:0x2:0x0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d30.sanity-pcc is idle now PASS 30 (8s) == sanity-pcc test 31: Test lctl pcc del command with --keep option ========================================================== 00:17:22 (1746159442) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127047 s, 413 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d31.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 65645331-857c-41d0-98f0-697237f91760 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f31.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=65645331-857c-41d0-98f0-697237f91760 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d31.sanity-pcc/d31.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d31.sanity-pcc/d31.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 30 autocache: projid={100} Cleanup PCC backend on /mnt/lustre /mnt/pcc.d31.sanity-pcc is idle now PASS 31 (8s) == sanity-pcc test 32: Test for RO-PCC when PCC copy is deleted ========================================================== 00:17:30 (1746159450) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100773 s, 520 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d32.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 41abb647-3c6d-413d-8bb2-6fb08d437418 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f32.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=41abb647-3c6d-413d-8bb2-6fb08d437418 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d32.sanity-pcc/d32.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f32.sanity-pcc vo_rc TMPPIPE=/tmp/multiop_open_wait_pipe.15544 client oleg217-client.virtnet multiop_bg started multiop_pid=15554 Stopping multiop_pid=15554 (kill 15554 on oleg217-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d32.sanity-pcc is idle now PASS 32 (13s) == sanity-pcc test 33: Cache rule with comparator (>, =, <) for file size ========================================================== 00:17:43 (1746159463) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151424 s, 346 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d33.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6beaa075-81f1-4be2-9589-90794a9aca1c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f33.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6beaa075-81f1-4be2-9589-90794a9aca1c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d33.sanity-pcc/d33.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.doc}&size<{1M} file: /mnt/lustre/myfile.doc, type: none file: /mnt/lustre/myfile.doc, type: none Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size<{5M}&size>{3M} Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size={5M 3M} Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d33.sanity-pcc is idle now PASS 33 (8s) == sanity-pcc test 34: Cache rule with comparator (>, <) for Project ID range ========================================================== 00:17:51 (1746159471) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory oleg217-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff88012a51c000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012d5be800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a51c000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012d5be800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697108 s, 752 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d34.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: eb10ffee-ce3a-4051-aed1-5b4951662a4f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f34.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=eb10ffee-ce3a-4051-aed1-5b4951662a4f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d34.sanity-pcc/d34.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid>{100} 99 - /mnt/lustre/f34.sanity-pcc 101 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{100} 102 - /mnt/lustre/f34.sanity-pcc 99 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{120}&projid>{110} 105 - /mnt/lustre/f34.sanity-pcc 121 - /mnt/lustre/f34.sanity-pcc 115 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d34.sanity-pcc is idle now PASS 34 (102s) == sanity-pcc test 35: mmap fault test =================== 00:19:33 (1746159573) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123355 s, 425 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d35.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f81de012-7c96-4004-9277-073bdffd39a1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f35.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f81de012-7c96-4004-9277-073bdffd39a1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d35.sanity-pcc/d35.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d35.sanity-pcc is idle now PASS 35 (7s) == sanity-pcc test 36a: Stale RW-PCC copy should be deleted after remove the PCC backend ========================================================== 00:19:40 (1746159580) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847855 s, 618 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b7b572d1-92ce-49f3-b76a-454bb1415214 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b7b572d1-92ce-49f3-b76a-454bb1415214 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a52' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc' check file 1:'0x200004a51:0x3:0x0' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52' Handle the file: 0x200004a51:0x3:0x0 Remove non-cached file: /mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52/0x200004a51:0x3:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36a.sanity-pcc is idle now PASS 36a (7s) == sanity-pcc test 36b: Stale RO-PCC copy should be deleted after remove the PCC backend ========================================================== 00:19:47 (1746159587) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102585 s, 511 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ca358382-0ed2-4803-8815-9750a7dcf894 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ca358382-0ed2-4803-8815-9750a7dcf894 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a54' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc' check file 1:'0x200004a51:0x5:0x0' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54' Handle the file: 0x200004a51:0x5:0x0 Remove non-cached file: /mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54/0x200004a51:0x5:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36b.sanity-pcc is idle now PASS 36b (6s) == sanity-pcc test 37: Multiple readers on a shared file with PCC-RO mode ========================================================== 00:19:53 (1746159593) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118434 s, 443 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d37.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 011f1023-bd36-4b0e-a63f-9b1567a72315 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=011f1023-bd36-4b0e-a63f-9b1567a72315 (extents) (64bit) (large files) (huge files) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108447 s, 483 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d37.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a20c13de-ab65-45c6-bdfa-d8ed63225fef Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=a20c13de-ab65-45c6-bdfa-d8ed63225fef (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d37.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} pcc: - pccpath: /mnt/pcc.d37.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} file: /mnt/lustre/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 file: /mnt/lustre2/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 /mnt/pcc.d37.sanity-pcc.2 is idle now /mnt/pcc.d37.sanity-pcc is idle now PASS 37 (10s) == sanity-pcc test 38: Verify LFS pcc state does not trigger prefetch for auto PCC-RO ========================================================== 00:20:03 (1746159603) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a993c000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012ab29800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a993c000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012ab29800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100275 s, 523 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d38.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8f4f3804-ce7c-499b-ba28-7886e5b3dc13 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f38.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8f4f3804-ce7c-499b-ba28-7886e5b3dc13 (extents) (64bit) (large files) (huge files) file: /mnt/lustre/d38.sanity-pcc/f38.sanity-pcc, type: none QQQQQ Cleanup PCC backend on /mnt/lustre /mnt/pcc.d38.sanity-pcc is idle now PASS 38 (95s) == sanity-pcc test 39: Test Project quota on loop PCC device ========================================================== 00:21:38 (1746159698) SKIP: sanity-pcc test_39 Not support project quota on local filesystem SKIP 39 (2s) == sanity-pcc test 40: Test async open attach in the background for PCC-RO file ========================================================== 00:21:40 (1746159700) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg217-client.virtnet /mnt/lustre (opts:) Stopping client oleg217-client.virtnet /mnt/lustre opts: Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg217-server Stopping /mnt/lustre-mds2 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg217-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg217-server tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) tune2fs 1.46.2.wc5 (26-Mar-2022) sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg217-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg217-server' oleg217-server: oleg217-server.virtnet: executing load_modules_local oleg217-server: Loading modules from /home/green/git/lustre-release/lustre oleg217-server: detected 4 online CPUs by sysfs oleg217-server: MODOPTS_LIBCFS= oleg217-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov /dev/mapper/mds1_flakey /mnt/lustre-mds1 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting mds2: -o localrecov /dev/mapper/mds2_flakey /mnt/lustre-mds2 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-MDT0001 Starting ost1: -o localrecov /dev/mapper/ost1_flakey /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov /dev/mapper/ost2_flakey /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg217-server: oleg217-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg217-client: oleg217-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Starting client oleg217-client.virtnet: -o user_xattr,flock 192.168.202.117@tcp:/lustre /mnt/lustre2 Started clients oleg217-client.virtnet: 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b5fc9000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012d5b8000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b5fc9000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012d5b8000.idle_timeout=debug disable quota as required 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.280567 s, 747 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d40.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: d90b8078-1d99-46b1-b9b9-41eb371a8e07 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f40.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d90b8078-1d99-46b1-b9b9-41eb371a8e07 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d40.sanity-pcc/d40.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.709456 s, 73.9 MB/s 100 - /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none ldlm.namespaces.lustre-OST0000-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012d5b8000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012d5b8000.lru_size=clear osc.lustre-OST0000-osc-ffff8800b5fc9000.stats=clear osc.lustre-OST0000-osc-ffff88012d5b8000.stats=clear osc.lustre-OST0001-osc-ffff8800b5fc9000.stats=clear osc.lustre-OST0001-osc-ffff88012d5b8000.stats=clear fail_val=1 fail_loc=0x214 llite.lustre-ffff8800b5fc9000.pcc_async_threshold=5MB llite.lustre-ffff88012d5b8000.pcc_async_threshold=5MB Test open attach with pcc_async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 2.07788 s, 505 kB/s Before: 0 After: 3 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none, flags: attaching /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 5s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1G llite.lustre-ffff88012d5b8000.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012d5b8000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012d5b8000.lru_size=clear osc.lustre-OST0000-osc-ffff8800b5fc9000.stats=clear osc.lustre-OST0000-osc-ffff88012d5b8000.stats=clear osc.lustre-OST0001-osc-ffff8800b5fc9000.stats=clear osc.lustre-OST0001-osc-ffff88012d5b8000.stats=clear Test open attach with async_threshold=1G file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: readonly, PCC_file: /d40.sanity-pcc/59f3/0x2000059f1:0x2:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0139632 s, 75.1 MB/s Before: 14 After: 14 Time1: 2 Time2: 6 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800b5fc9000.pcc_async_threshold=5MB llite.lustre-ffff88012d5b8000.pcc_async_threshold=5MB ldlm.namespaces.lustre-OST0000-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012d5b8000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012d5b8000.lru_size=clear Read 1MB data with async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 1.06667 s, 983 kB/s /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 5s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1G llite.lustre-ffff88012d5b8000.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012d5b8000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012d5b8000.lru_size=clear Read 1MB data with async_threshold=1G 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0088446 s, 119 MB/s Time1: 1 Time2: 6 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d40.sanity-pcc is idle now PASS 40 (130s) == sanity-pcc test 41: Test mtime rule for PCC-RO open attach with O_RDONLY mode ========================================================== 00:23:50 (1746159830) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107775 s, 486 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d41.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 70b46cba-c705-4cb0-a55c-d1955e9bcaf0 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f41.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=70b46cba-c705-4cb0-a55c-d1955e9bcaf0 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d41.sanity-pcc/d41.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: mtime>{1m} pcc_ro_data pcc_ro_data Cleanup PCC backend on /mnt/lustre /mnt/pcc.d41.sanity-pcc is idle now PASS 41 (7s) == sanity-pcc test 42: PCC attach without attach ID specified ========================================================== 00:23:57 (1746159837) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.181183 s, 347 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d42.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: e3484fdf-d9a9-4012-b120-bb662b952832 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f42.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e3484fdf-d9a9-4012-b120-bb662b952832 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d42.sanity-pcc/d42.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} file: /mnt/lustre/f42.sanity-pcc, type: readonly, PCC_file: /d42.sanity-pcc/59f5/0x2000059f1:0x4:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d42.sanity-pcc is idle now PASS 42 (7s) == sanity-pcc test 43: Auto attach at open() should add capacity owner check ========================================================== 00:24:04 (1746159844) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.120245 s, 523 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d43.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: a2a0ce68-504e-4ff7-8eec-55c951db7739 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f43.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a2a0ce68-504e-4ff7-8eec-55c951db7739 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d43.sanity-pcc/d43.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: size<{100M} file: /mnt/lustre/f43.sanity-pcc, type: none running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f43.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f43.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/f43.sanity-pcc] file: /mnt/lustre/f43.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d43.sanity-pcc is idle now PASS 43 (6s) == sanity-pcc test 44: Verify valid auto attach without re-fetching the whole files ========================================================== 00:24:10 (1746159850) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.200196 s, 314 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d44.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 63ec39e0-4b50-459e-b1d5-b59f851c6a88 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f44.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=63ec39e0-4b50-459e-b1d5-b59f851c6a88 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d44.sanity-pcc/d44.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800b5fc9000.pcc_async_threshold=0 llite.lustre-ffff88012d5b8000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.867831 s, 60.4 MB/s /mnt/lustre/f44.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 388 0x184 0x280000401 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.609674 s, 86.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.690229 s, 76.0 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.684006 s, 76.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.621606 s, 84.3 MB/s 52428800 bytes (52 MB) copied, 0.629066 s, 83.3 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.635536 s, 82.5 MB/s 52428800 bytes (52 MB) copied, 0.658089 s, 79.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.689431 s, 76.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.639639 s, 82.0 MB/s 52428800 bytes (52 MB) copied, 0.641581 s, 81.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.714648 s, 73.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.73278 s, 71.5 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.653033 s, 80.3 MB/s 52428800 bytes (52 MB) copied, 0.673608 s, 77.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.737445 s, 71.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.715669 s, 73.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453055 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647127 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683995 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527591 s, 994 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269538 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0956862 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562505 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0887099 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136035 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365346 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540402 s, 970 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729079 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359615 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436701 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.108721 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796785 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557062 s, 941 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0207395 s, 2.5 GB/s , 0.0310818 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427986 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284517 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446582 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407785 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497685 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0981091 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838908 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909195 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473287 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498062 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567309 s, 924 MB/s 52428800 bytes (52 MB) copied, 0.0621536 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394993 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534458 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346982 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228819 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531423 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755316 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030419 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486801 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641401 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0968949 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563186 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108952 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112676 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704371 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815386 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0950947 s, 551 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576386 s, 910 MB/s 52428800 bytes (52 MB) copied, 0.118603 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154857 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189352 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15971 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192316 s, 273 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118034 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120577 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121943 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181355 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612279 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798782 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783205 s, 669 MB/s , 0.0704153 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565595 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598599 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819869 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486618 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822387 s, 638 MB/s , 0.0766942 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873656 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044192 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985773 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731292 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748738 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036834 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460043 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583106 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027723 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580006 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860585 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106354 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958194 s, 547 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120126 s, 436 MB/s 52428800 bytes (52 MB) copied, 0.082998 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590183 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470576 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0999884 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689014 s, 761 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0595998 s, 880 MB/s , 0.0508396 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107646 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281301 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544398 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666017 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114761 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108774 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116024 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907291 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910032 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849149 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121901 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196044 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0940567 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23379 s, 224 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.257556 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829327 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23693 s, 221 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208473 s, 251 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10006 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763539 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619572 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103322 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120811 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805674 s, 651 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110962 s, 472 MB/s 52428800 bytes (52 MB) copied, 0.103074 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127427 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769471 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1251 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114629 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157031 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11684 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806589 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848157 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980516 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612653 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709599 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099704 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691406 s, 758 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0577054 s, 909 MB/s , 0.0505442 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658455 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05297 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663736 s, 790 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049831 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069929 s, 750 MB/s 52428800 bytes (52 MB) copied, 0.096564 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793803 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764003 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132024 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117492 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137338 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245441 s, 214 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.287263 s, 183 MB/s 52428800 bytes (52 MB) copied, 0.168396 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782053 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149937 s, 350 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829645 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137846 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817128 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0950975 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160439 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112943 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539928 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136767 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597924 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851403 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127564 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136107 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206234 s, 254 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126985 s, 413 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278651 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143982 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242985 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.131671 s, 398 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.271633 s, 193 MB/s , 0.205466 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084817 s, 618 MB/s 52428800 bytes (52 MB) copied, 0.148631 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130921 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773763 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589067 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0962693 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0878505 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242861 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636003 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550256 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353179 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725467 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033267 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676351 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594144 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143124 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149392 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163125 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126989 s, 413 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116923 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10765 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118109 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115702 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147639 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129324 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263087 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150219 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499787 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161887 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196941 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258192 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148312 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125116 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537917 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824912 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997152 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703313 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832875 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886757 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738682 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0991704 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104838 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162573 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773315 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476938 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116453 s, 450 MB/s 52428800 bytes (52 MB) copied, 0.103411 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129497 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075563 s, 694 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164035 s, 320 MB/s 52428800 bytes (52 MB) copied, 0.195078 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105855 s, 495 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.225029 s, 233 MB/s , 0.156154 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178165 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295697 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579315 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036824 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634259 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754986 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280374 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124017 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120876 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674364 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485034 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462887 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838643 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113485 s, 462 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610118 s, 859 MB/s 52428800 bytes (52 MB) copied, 0.124709 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615953 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234204 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651989 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704824 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418607 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933445 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162287 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184953 s, 283 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196961 s, 266 MB/s 52428800 bytes (52 MB) copied, 0.168257 s, 312 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160502 s, 327 MB/s 52428800 bytes (52 MB) copied, 0.177768 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155143 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10807 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070381 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263827 s, 199 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184583 s, 284 MB/s 52428800 bytes (52 MB) copied, 0.195454 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174392 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440671 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160697 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.220709 s, 238 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981461 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814075 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050349 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367622 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045569 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708015 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0751141 s, 698 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0776531 s, 675 MB/s , 0.0733468 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391248 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100154 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371822 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10366 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982551 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0986649 s, 531 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237545 s, 221 MB/s 52428800 bytes (52 MB) copied, 0.119165 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181685 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13069 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158044 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243682 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245387 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.313552 s, 167 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134685 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301537 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.393327 s, 133 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318305 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265757 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14343 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193817 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270354 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840787 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882737 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121672 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172514 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409111 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463974 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684824 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740318 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060642 s, 865 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454222 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783175 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793891 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137333 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303811 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482358 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0593256 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565839 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0955012 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657185 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061492 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726846 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785032 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108726 s, 482 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610508 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769232 s, 682 MB/s 52428800 bytes (52 MB) copied, 0.0739472 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881912 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06762 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699083 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598951 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882719 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678179 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373184 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634485 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109565 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757092 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606937 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721092 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779819 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568396 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616588 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063146 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634022 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04072 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851869 s, 615 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101452 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118505 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547468 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0914021 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110732 s, 473 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124486 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390392 s, 1.3 GB/s , 0.104224 s, 503 MB/s 52428800 bytes (52 MB) copied, 0.101176 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0938912 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637133 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756563 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491255 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711644 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0884951 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909427 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0859356 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0905047 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142636 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141921 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126941 s, 413 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752593 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188741 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137767 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626949 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17444 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174026 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785554 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171756 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139617 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311127 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183009 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703406 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108249 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092378 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.241297 s, 217 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163528 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136623 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126743 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183343 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136637 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181854 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158132 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11178 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558182 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777107 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11561 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101532 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235439 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478269 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075058 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541192 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468237 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547069 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440328 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216687 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123342 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15505 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392552 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648678 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649358 s, 807 MB/s 52428800 bytes (52 MB) copied, 0.136831 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113415 s, 462 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114974 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138408 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176095 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626846 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117368 s, 447 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725747 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143589 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342066 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481099 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401362 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0392922 s, 1.3 GB/s , 0.0689749 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679952 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408075 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698559 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0930043 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756889 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108093 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.097128 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343634 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779632 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166099 s, 316 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111272 s, 471 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109338 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123403 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366102 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492506 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628583 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833269 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520998 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077068 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448672 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325507 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377395 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699686 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551927 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726992 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948665 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931998 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102428 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129521 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136752 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149042 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12503 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187644 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118856 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137926 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725491 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581985 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103598 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718051 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557092 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375856 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974021 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900632 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826973 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809083 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815207 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520018 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135572 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308277 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722935 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570168 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775991 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0862755 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656699 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136966 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149171 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0945136 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143886 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110366 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24387 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150403 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14464 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161907 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027037 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832681 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136896 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117519 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647609 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496185 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156596 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408401 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047047 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717923 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462404 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067345 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836515 s, 627 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118379 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920578 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438878 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0912891 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848045 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742523 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863318 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282723 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752824 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11494 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305391 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112102 s, 468 MB/s , 0.0780221 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966886 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686963 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114262 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876704 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07092 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830155 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707233 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535768 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188641 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903695 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448904 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349711 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309302 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728891 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616104 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556281 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719767 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942309 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453835 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381224 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578817 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104488 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949411 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113346 s, 463 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104469 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131207 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607199 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763962 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12029 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072821 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139044 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127505 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156666 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122411 s, 428 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332178 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599864 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975717 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775576 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642546 s, 816 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111995 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564075 s, 929 MB/s 52428800 bytes (52 MB) copied, 0.115095 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16451 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770734 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185726 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117104 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114642 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735117 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101456 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715452 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418138 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511321 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405796 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780986 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642254 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498615 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723314 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100877 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0991245 s, 529 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683972 s, 767 MB/s 52428800 bytes (52 MB) copied, 0.0814987 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658087 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753723 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119786 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141293 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025913 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517064 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389221 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061585 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727322 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239881 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414956 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056212 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558317 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619084 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118195 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579372 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445754 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994047 s, 527 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0401692 s, 1.3 GB/s , 0.0534361 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749446 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439545 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697748 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103659 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104347 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413405 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182369 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16026 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105261 s, 498 MB/s 52428800 bytes (52 MB) copied, 0.0633576 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291958 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106629 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592528 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023972 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805427 s, 651 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056343 s, 931 MB/s 52428800 bytes (52 MB) copied, 0.0510429 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157047 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652727 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452664 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653581 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042335 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039287 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356781 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560916 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504451 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527204 s, 994 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522242 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624095 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814718 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639044 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688358 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540369 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640493 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671384 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0899404 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454273 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0878123 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817595 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946037 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632791 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708132 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588026 s, 892 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961265 s, 545 MB/s 52428800 bytes (52 MB) copied, 0.0586608 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387886 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503894 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611502 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291847 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472753 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326747 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467842 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433933 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332968 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564255 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511176 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469443 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449876 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700563 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637202 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799993 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100225 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0915438 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139044 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188934 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830537 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17573 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243287 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15022 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166077 s, 316 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171933 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954295 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.31944 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120806 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25861 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291183 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270197 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103935 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285524 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10965 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108161 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643581 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824101 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212336 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583288 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668615 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760049 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075935 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780133 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781058 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649966 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724923 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402054 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044789 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483022 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570634 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447781 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549409 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707354 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528038 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290377 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430878 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238269 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618294 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640521 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047152 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521688 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116813 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840532 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707537 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685502 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334161 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444431 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785126 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443153 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105637 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488546 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886957 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0862748 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082238 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836313 s, 627 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03697 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565357 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101358 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0977725 s, 536 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120149 s, 436 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186633 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122967 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153005 s, 343 MB/s 52428800 bytes (52 MB) copied, 0.102695 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865272 s, 606 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14738 s, 356 MB/s 52428800 bytes (52 MB) copied, 0.201835 s, 260 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038264 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448408 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725911 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871833 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128245 s, 409 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705841 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603067 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720255 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749259 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044598 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734582 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104333 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421183 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732103 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115277 s, 455 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119284 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1354 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109213 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463736 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635689 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528457 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890562 s, 589 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0591372 s, 887 MB/s , 0.101637 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850583 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492189 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616247 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433923 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894777 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645432 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698169 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631836 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745283 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503155 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512238 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724243 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570696 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853262 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067162 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638898 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110814 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860016 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279151 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517122 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648886 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596131 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849086 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794291 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093915 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794473 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115205 s, 455 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118271 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106051 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359657 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452117 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046742 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030907 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482716 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400575 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796981 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989309 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909952 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309469 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563661 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036095 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359805 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398015 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391258 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539053 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461857 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299451 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391633 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612825 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619053 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437511 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038936 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544776 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754247 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740408 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405124 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620387 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435699 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523238 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682104 s, 769 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482061 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121595 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124229 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891797 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891463 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0955799 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886725 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488355 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574501 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133784 s, 392 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115396 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384789 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065121 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534437 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595948 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434096 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799404 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838926 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105879 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783518 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880632 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614358 s, 853 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779864 s, 672 MB/s 52428800 bytes (52 MB) copied, 0.0798974 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07814 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665018 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891572 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074973 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312189 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366298 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107366 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072384 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484876 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710289 s, 738 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700921 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886246 s, 592 MB/s 52428800 bytes (52 MB) copied, 0.0706872 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678333 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933192 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121096 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111997 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0792234 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117011 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100087 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722958 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468821 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659605 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761377 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567707 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422015 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457914 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270021 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152294 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112122 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693025 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628508 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289248 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304419 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433687 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382003 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526141 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506594 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360251 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493983 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116607 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910314 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811618 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835299 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762974 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766529 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701101 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100997 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136813 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175952 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533972 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22166 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947387 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350417 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197753 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225785 s, 232 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135091 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705082 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647775 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334684 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438123 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066561 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385138 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584122 s, 898 MB/s , 0.0651448 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0862163 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108909 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983923 s, 533 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815655 s, 643 MB/s 52428800 bytes (52 MB) copied, 0.0794728 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633969 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065793 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104507 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735127 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058019 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088809 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454577 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305039 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349145 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453239 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495371 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532433 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767286 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706301 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440533 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100523 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110868 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758758 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056091 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650379 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629544 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746585 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110872 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450448 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857935 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765772 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640749 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755863 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591955 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433326 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371248 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434803 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487429 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441377 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591394 s, 887 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584647 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436842 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.059087 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114095 s, 460 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795183 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07246 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118989 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439738 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253281 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638491 s, 821 MB/s 52428800 bytes (52 MB) copied, 0.0938511 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110215 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084141 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849294 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622202 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0921658 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797008 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404836 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548838 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650279 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101186 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103532 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868729 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155883 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980287 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411419 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118824 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934037 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205045 s, 256 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942959 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493538 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.110518 s, 474 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802787 s, 653 MB/s 52428800 bytes (52 MB) copied, 0.127158 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803476 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703211 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795872 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533614 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459628 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482126 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531732 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041356 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570467 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401447 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048711 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440173 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519555 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689432 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424899 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304836 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376399 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640522 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347985 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229335 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677106 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502001 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472797 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563201 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092596 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802173 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119216 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040966 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275162 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644188 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112991 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490656 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689395 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716237 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574266 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801909 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671444 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603062 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478066 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05733 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488654 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300348 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745872 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542695 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484438 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100281 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070223 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641399 s, 817 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.134232 s, 391 MB/s , 0.114863 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437042 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850222 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069154 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475125 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699331 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025326 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677137 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423333 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586711 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463013 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386407 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510303 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357271 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0391719 s, 1.3 GB/s , 0.0285195 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374358 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460652 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436136 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506574 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064229 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564154 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583889 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065674 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583625 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621022 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463275 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723606 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100734 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108169 s, 485 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158286 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136323 s, 385 MB/s 52428800 bytes (52 MB) copied, 0.208818 s, 251 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157239 s, 333 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.14677 s, 357 MB/s , 0.146967 s, 357 MB/s 52428800 bytes (52 MB) copied, 0.115327 s, 455 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133518 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173319 s, 302 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131667 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167022 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197542 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0896467 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424443 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542273 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338799 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605591 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521409 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918167 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514195 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563936 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066952 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685162 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442886 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067025 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288133 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253808 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662603 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0990618 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104318 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475238 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132735 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127622 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651121 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847802 s, 618 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0635878 s, 825 MB/s 50+0 records in 50+0 records out , 0.0505795 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0855624 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569234 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632083 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903353 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404942 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472799 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622007 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491916 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505316 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540431 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385652 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466205 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591697 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839476 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886673 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14207 s, 369 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748685 s, 700 MB/s 52428800 bytes (52 MB) copied, 0.131105 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624968 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145081 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10463 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136049 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114748 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966115 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549937 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746562 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0926362 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121022 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853463 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10531 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139176 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10378 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134911 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152293 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837907 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904034 s, 580 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464995 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0493709 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069264 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578355 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481607 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0753476 s, 696 MB/s , 0.0514223 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791215 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270253 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595422 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424819 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276583 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707836 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360399 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519046 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397732 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661023 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531336 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689232 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775439 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451239 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465722 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700578 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390857 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386016 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062289 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696042 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513795 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582419 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502582 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489091 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461156 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465768 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386577 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250867 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450166 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405828 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489583 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329705 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877472 s, 597 MB/s 52428800 bytes (52 MB) copied, 0.0773057 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949325 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672364 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054694 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116676 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900072 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534759 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666619 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0834061 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1026 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103728 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358332 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588525 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717563 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482994 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315694 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539015 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072165 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413274 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120519 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426953 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333658 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546624 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02364 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390207 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377175 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254218 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620476 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382353 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480719 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609438 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775716 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334843 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464269 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688539 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573734 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528599 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492872 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054731 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429943 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280158 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304732 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279638 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597125 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918045 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736005 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879156 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933291 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131222 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149022 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855532 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167849 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141838 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130636 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141199 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445535 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120103 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16314 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518701 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0811656 s, 646 MB/s , 0.0440876 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858188 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0787223 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624283 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324449 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450392 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600364 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462037 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043389 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433277 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483338 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581607 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574757 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407431 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347529 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614307 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697297 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107925 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080926 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499766 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827303 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860037 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525171 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665187 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051871 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.08914 s, 588 MB/s , 0.0602933 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657617 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079207 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10762 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093349 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139959 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174156 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102176 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609145 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137054 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518664 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664437 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762229 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101703 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810847 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623087 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522815 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073321 s, 715 MB/s 52428800 bytes (52 MB) copied, 0.0800537 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456895 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042101 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617055 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072249 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352775 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793366 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486757 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546094 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590195 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044088 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700364 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794104 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.087983 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877247 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163762 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188102 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131621 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280056 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290749 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.372268 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187297 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192021 s, 273 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.317998 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33118 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213037 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184904 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409933 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140289 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146674 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903137 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104011 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140889 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141984 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681163 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997439 s, 526 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139858 s, 375 MB/s 52428800 bytes (52 MB) copied, 0.154585 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858861 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057872 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629481 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440392 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510706 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546489 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641647 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749854 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718062 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798014 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743782 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779759 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0787406 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739272 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840321 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0842902 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653353 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685329 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556788 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501959 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619466 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0986314 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954581 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448328 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538097 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547149 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660524 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537045 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749759 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806262 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114684 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547117 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591907 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493301 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412948 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429976 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296146 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0553676 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662408 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063231 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308897 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737735 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369133 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368431 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423581 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296581 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850876 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521796 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408489 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418051 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0936444 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0990857 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444242 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538823 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679963 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0751532 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390039 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848307 s, 618 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922656 s, 568 MB/s 52428800 bytes (52 MB) copied, 0.0792414 s, 662 MB/s 52428800 bytes (52 MB) copied, 0.105584 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985565 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573628 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920021 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855986 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322988 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406705 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533054 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559362 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400674 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512593 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319483 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047114 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479226 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729948 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660536 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683054 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455289 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591286 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391995 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313188 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404961 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550932 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105762 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967839 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794224 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436156 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518355 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459206 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741733 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305396 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463436 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544888 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448155 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610378 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545839 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480574 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452338 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869508 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443205 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434964 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745241 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577308 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644729 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287981 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452167 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110953 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923857 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828425 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239486 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639116 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637878 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583972 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658767 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300626 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0911132 s, 575 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493989 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493198 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194353 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285324 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14944 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785211 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530525 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530151 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368038 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091489 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813161 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484851 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740171 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886631 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745916 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514238 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0834101 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398236 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039892 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410901 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338617 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479938 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840491 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389155 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469087 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503402 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463632 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755679 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443701 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072119 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375929 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356227 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362951 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810639 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373388 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101127 s, 518 MB/s 52428800 bytes (52 MB) copied, 0.104602 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107892 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485525 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454637 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609788 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231126 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477472 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361658 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847236 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806204 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892024 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048628 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356772 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213849 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697287 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857044 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813787 s, 644 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191284 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426907 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0927975 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364851 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0939762 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382851 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365528 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521373 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576398 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567355 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503557 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654744 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652961 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717402 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344306 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686715 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203387 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376382 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072024 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525939 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05176 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386427 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746056 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101773 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157852 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760135 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767647 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794407 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650455 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609945 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433441 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039686 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071427 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521207 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739148 s, 709 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710253 s, 738 MB/s 52428800 bytes (52 MB) copied, 0.0834342 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416722 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029124 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647939 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620857 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649889 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521543 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750978 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533138 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850412 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821218 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403359 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924164 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672781 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287901 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585279 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047365 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202229 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022064 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245095 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249312 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873471 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515661 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809143 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655062 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526414 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031522 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405343 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579583 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587338 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071141 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651789 s, 804 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543785 s, 964 MB/s 52428800 bytes (52 MB) copied, 0.0609969 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432531 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437845 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062706 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424671 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685262 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274124 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326703 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351676 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647905 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643935 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515621 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606785 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929641 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538071 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647688 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770839 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068232 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547107 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765777 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726482 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107309 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309999 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553892 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575005 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715825 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611416 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519492 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738008 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0930243 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052309 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890438 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917559 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0842906 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428454 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868642 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0984623 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437314 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495856 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545429 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230923 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054146 s, 968 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490825 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0423848 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351704 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714413 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372879 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351244 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531381 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056073 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425263 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568803 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716086 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638497 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107981 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055229 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498627 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964221 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994167 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469807 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200762 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627227 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135295 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14354 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180308 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462161 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178239 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05535 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582178 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383894 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588488 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0874502 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759101 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382513 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042371 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072818 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543832 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103661 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101869 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140476 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077275 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0963897 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107547 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858655 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814203 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697675 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094443 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158422 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521853 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450755 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073457 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032825 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375729 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648279 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534017 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686637 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744241 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518199 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725146 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763712 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966217 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809914 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0841477 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532371 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501173 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225348 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435254 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0914338 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946498 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460756 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106885 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573158 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059013 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059453 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652329 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225481 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102017 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806094 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760969 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561746 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340816 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0965682 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646703 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346663 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778603 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322814 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728008 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413511 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380393 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510933 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439949 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10839 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258266 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530198 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399216 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559869 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062523 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544635 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853877 s, 614 MB/s , 0.121386 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982269 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586782 s, 893 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.109118 s, 480 MB/s , 0.101696 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880753 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601276 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119863 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172388 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108571 s, 483 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985646 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840219 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767006 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791041 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947744 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766922 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411215 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494115 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835426 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377357 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777493 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164511 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933276 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138646 s, 378 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161435 s, 325 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.092571 s, 566 MB/s , 0.133156 s, 394 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662057 s, 792 MB/s 52428800 bytes (52 MB) copied, 0.0610261 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656396 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299816 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546115 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918984 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537269 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044466 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332496 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775005 s, 676 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778082 s, 674 MB/s 52428800 bytes (52 MB) copied, 0.11536 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.096961 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0991343 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149019 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123427 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118322 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154537 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071002 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786908 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103075 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067198 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906152 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533118 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1635 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0733051 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170665 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084198 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540998 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404602 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621647 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383128 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357969 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068173 s, 769 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368136 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577232 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588994 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778558 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076518 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155848 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259669 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351055 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494376 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019871 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031026 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639079 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595527 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613169 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053583 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0896877 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648362 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553807 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556349 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116618 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969835 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764181 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346348 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075476 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105275 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401064 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633448 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725128 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138026 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095782 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429168 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554713 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775296 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642699 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566091 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300984 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054886 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040147 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579098 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373149 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423854 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595295 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546207 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665396 s, 788 MB/s , 0.0661912 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352938 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397625 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268549 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465915 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509126 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653283 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262468 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721339 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602464 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292343 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594962 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750581 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426905 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641645 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354838 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677269 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486162 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035906 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688427 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494827 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820107 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175607 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101397 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10641 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111767 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100712 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128314 s, 409 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0940558 s, 557 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108744 s, 482 MB/s 52428800 bytes (52 MB) copied, 0.112535 s, 466 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103743 s, 505 MB/s 52428800 bytes (52 MB) copied, 0.0906998 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106622 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371361 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559992 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067813 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209502 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253842 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934196 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520572 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135286 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446366 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961237 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265843 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109335 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382589 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307217 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481545 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0747227 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381423 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0972593 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890716 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117576 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838444 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681372 s, 769 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082022 s, 639 MB/s 52428800 bytes (52 MB) copied, 0.0595779 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989719 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967612 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511189 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754427 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763342 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350382 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239045 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376708 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573439 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563599 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350713 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618177 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543123 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705233 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605093 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975905 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136928 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250587 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159116 s, 330 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314004 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16175 s, 324 MB/s 52428800 bytes (52 MB) copied, 0.0350814 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718122 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234354 s, 224 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172129 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105271 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707524 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659141 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702036 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260869 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0914265 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758926 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494446 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463336 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440561 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0886012 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116729 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555071 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381184 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601581 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571243 s, 918 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686936 s, 763 MB/s 52428800 bytes (52 MB) copied, 0.115879 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981236 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108388 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0908883 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967047 s, 542 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120126 s, 436 MB/s 52428800 bytes (52 MB) copied, 0.107823 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604195 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120689 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298948 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551332 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585247 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545917 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100969 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107111 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496795 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122664 s, 427 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115843 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481677 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039203 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0884778 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102878 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460413 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386092 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532462 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071004 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716419 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0867193 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819381 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0899367 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621762 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565422 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780717 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057828 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893001 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540123 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654925 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053449 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873858 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585406 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338162 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283234 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232161 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568728 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222206 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350432 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334185 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060937 s, 860 MB/s 52428800 bytes (52 MB) copied, 0.0474993 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523074 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756711 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715117 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677048 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989442 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127341 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334067 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351627 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646847 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596148 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035408 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798601 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227727 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397706 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105216 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580472 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669976 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574584 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477132 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393383 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480551 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555695 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846147 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073952 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332186 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123727 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171523 s, 306 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824142 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102678 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073356 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03962 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496967 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551526 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434201 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419984 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119681 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107313 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120507 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877719 s, 597 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1061 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756774 s, 693 MB/s 52428800 bytes (52 MB) copied, 0.133687 s, 392 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149829 s, 350 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983454 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860056 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118111 s, 444 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0870656 s, 602 MB/s , 0.15302 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943267 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410398 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396146 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455926 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489344 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848922 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754228 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072225 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890984 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475995 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394367 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247767 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651085 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425308 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0344865 s, 1.5 GB/s , 0.0606243 s, 865 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642491 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466779 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602797 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407302 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459286 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599262 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488205 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286456 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0401105 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399526 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327314 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333073 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310163 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390393 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344445 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742527 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604457 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806773 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903288 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534764 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579968 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594868 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519536 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371908 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430664 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579613 s, 905 MB/s 52428800 bytes (52 MB) copied, 0.0372505 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396821 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534054 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319267 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746865 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891632 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279924 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588037 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414354 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714258 s, 734 MB/s 52428800 bytes (52 MB) copied, 0.0828342 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034449 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035716 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382385 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479797 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475267 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556302 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536962 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736491 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735851 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11035 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954438 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799587 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699406 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11645 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161277 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0936729 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322614 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439103 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397238 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435485 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538364 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490606 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400894 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366092 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657881 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826706 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131895 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758371 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666422 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589386 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888766 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526848 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941729 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881526 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135255 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142508 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142299 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098036 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1562 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820177 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892921 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992879 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064935 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630662 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857842 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446702 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516908 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347259 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547682 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347424 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621238 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528337 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647658 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789246 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0790029 s, 664 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857102 s, 612 MB/s 52428800 bytes (52 MB) copied, 0.104488 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115636 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104528 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678917 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165492 s, 317 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687687 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125816 s, 417 MB/s 52428800 bytes (52 MB) copied, 0.0830058 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983788 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491092 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0498359 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.073169 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721103 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444127 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424631 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617125 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564259 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340908 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0713434 s, 735 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081417 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101763 s, 515 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0579645 s, 904 MB/s , 0.0889802 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542311 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589436 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427593 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431743 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639775 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.097881 s, 536 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0854742 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595097 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325088 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374871 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495796 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740517 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355762 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651651 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828843 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906243 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135188 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550737 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036406 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387498 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489965 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537437 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431132 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477409 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296531 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404924 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361301 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048194 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837206 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946865 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376076 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482651 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063249 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386161 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871343 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0914528 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848112 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860467 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167845 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113303 s, 463 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137884 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548322 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181599 s, 289 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791099 s, 663 MB/s 52428800 bytes (52 MB) copied, 0.0677152 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691476 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805768 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399801 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422676 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345822 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524791 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426554 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588795 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424517 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652549 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076666 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079401 s, 660 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576961 s, 909 MB/s 52428800 bytes (52 MB) copied, 0.124955 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803974 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537477 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607889 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640054 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556373 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058651 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641896 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439199 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375364 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491308 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382221 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561026 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245312 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347714 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0545514 s, 961 MB/s , 0.0532075 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375993 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309207 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591543 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383509 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621928 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118122 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388676 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157877 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550568 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693678 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941184 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460903 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524458 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585066 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440827 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447893 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476066 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697326 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726274 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081048 s, 647 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881864 s, 595 MB/s 52428800 bytes (52 MB) copied, 0.0812874 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471825 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461097 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849915 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258733 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542629 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476695 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509704 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515141 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377529 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425594 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492245 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299116 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346978 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377536 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284797 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321278 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511816 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043014 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408977 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290188 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271974 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539671 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678444 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346753 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276914 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568447 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319087 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387967 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280966 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641023 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0804779 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601379 s, 872 MB/s , 0.0589654 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337405 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356981 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451299 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0823002 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403529 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541868 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744725 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684661 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386833 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546167 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687694 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476379 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498483 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659362 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529669 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518731 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461129 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437955 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048521 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572671 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343574 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406437 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517799 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258049 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300873 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430429 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051402 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329003 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235428 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579279 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543376 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284559 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044512 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479008 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121115 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515213 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438603 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880817 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154804 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121496 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423889 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690526 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719754 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055043 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478836 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429488 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436272 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435442 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02755 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539937 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0970921 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508167 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543026 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708513 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230386 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477237 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283711 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587986 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826137 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562978 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402063 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469037 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810068 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714616 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323058 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906673 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624983 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420962 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421123 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696167 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220619 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282599 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629352 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255486 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422784 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343332 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661719 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435554 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544325 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257994 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508991 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363017 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744606 s, 704 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719874 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513015 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0706169 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789805 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237234 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635194 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060317 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368926 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711985 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410738 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223577 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373311 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237945 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371673 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401711 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034218 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618391 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369775 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373801 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473497 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522731 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379006 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572212 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717872 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480209 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0940069 s, 558 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.097495 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295274 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.109485 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654131 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11156 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618463 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043059 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564957 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691805 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663888 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433932 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463216 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665783 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312873 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489354 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332226 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715413 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033057 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706404 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577527 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268797 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469259 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395786 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0583707 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359133 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847178 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250093 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649103 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421643 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065361 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502142 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582698 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414831 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329021 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038058 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049154 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594721 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614054 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241676 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775304 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737904 s, 711 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065013 s, 806 MB/s 52428800 bytes (52 MB) copied, 0.0796949 s, 658 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078122 s, 671 MB/s 52428800 bytes (52 MB) copied, 0.100327 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640132 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111016 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056283 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600355 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585675 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0986359 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648068 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554618 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338951 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053495 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599101 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02584 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259063 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816286 s, 642 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128182 s, 409 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152273 s, 344 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638448 s, 821 MB/s 52428800 bytes (52 MB) copied, 0.0688216 s, 762 MB/s 52428800 bytes (52 MB) copied, 0.0858354 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580531 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634928 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14178 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474941 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033685 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102824 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476015 s, 1.1 GB/s , 0.153482 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725762 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590991 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566106 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349904 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458041 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319139 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473225 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448447 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455145 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839584 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640999 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020835 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622377 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440809 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105555 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150565 s, 348 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773202 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666795 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469601 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187921 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532192 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710095 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647183 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494055 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637326 s, 823 MB/s 52428800 bytes (52 MB) copied, 0.0721519 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803026 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054623 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118013 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101119 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289814 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317558 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100126 s, 524 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764819 s, 686 MB/s 52428800 bytes (52 MB) copied, 0.077336 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175494 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869635 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455154 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107937 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891521 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0986919 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580052 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447299 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490323 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323989 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391159 s, 1.3 GB/s , 0.0598639 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571519 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431201 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324261 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166747 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608793 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490132 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481836 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621628 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498133 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503147 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319141 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764756 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538453 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384195 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370877 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381334 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711368 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583619 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668795 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0548484 s, 956 MB/s 52428800 bytes (52 MB) copied, 0.0760409 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100354 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110456 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734658 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616525 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931882 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0867702 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344345 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919202 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137824 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13532 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268638 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431886 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027552 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277235 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365966 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279204 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0277828 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559415 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800336 s, 655 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052281 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0309483 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431831 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293365 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442063 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041129 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299181 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309492 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334755 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352517 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03265 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520384 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389096 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624613 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451528 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321083 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554514 s, 945 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275562 s, 1.9 GB/s , 0.0319287 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0572543 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426441 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026167 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212048 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243756 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664293 s, 789 MB/s Finish ======== /mnt/lustre/f44.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0x184:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800b5fc9000.stats= snapshot_time 1746159913.992029097 secs.nsecs start_time 1746159853.595531886 secs.nsecs elapsed_time 60.396497211 secs.nsecs read_bytes 1402 samples [bytes] 315392 1048576 1468006400 1538381351747584 read 127250 samples [usecs] 157 546810 192209241 4787236098251 ioctl 1 samples [reqs] open 2548 samples [usecs] 5 109904 4237357 91012933845 close 2548 samples [usecs] 9 56589 2586185 40591278603 seek 2544 samples [usecs] 2 4324 27024 35680562 setattr 1 samples [usecs] 29030 29030 29030 842740900 getattr 17 samples [usecs] 141 6653 25077 126040365 mknod 1 samples [usecs] 7296 7296 7296 53231616 inode_permission 7895 samples [usecs] 1 40390 5067685 58247676693 opencount 2549 samples [reqs] 1 2547 3246126 5512439480 openclosetime 2547 samples [usecs] 6 216106 57621434 2912758052570 pcc_attach 1 samples [reqs] pcc_detach 237 samples [reqs] pcc_auto_attach 236 samples [reqs] pcc_hit_bytes 2517 samples [bytes] 52428800 52428800 131963289600 6918676917780480000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff88012d5b8000.stats= snapshot_time 1746159913.992156250 secs.nsecs start_time 1746159853.595554350 secs.nsecs elapsed_time 60.396601900 secs.nsecs attach 1 detach 237 auto_attach 236 llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1073741824 llite.lustre-ffff88012d5b8000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d44.sanity-pcc is idle now PASS 44 (67s) == sanity-pcc test 45: Concurrent read access from two mount points ========================================================== 00:25:17 (1746159917) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.13698 s, 765 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d45.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: f48ad2d8-f21d-4cfa-946e-a0118a7eed47 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f48ad2d8-f21d-4cfa-946e-a0118a7eed47 (extents) (64bit) (large files) (huge files) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.145271 s, 722 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d45.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: d11cdf0a-db71-4535-bccd-a3f20e7c00c2 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=d11cdf0a-db71-4535-bccd-a3f20e7c00c2 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d45.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} pcc: - pccpath: /mnt/pcc.d45.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800b5fc9000.pcc_async_threshold=0 llite.lustre-ffff88012d5b8000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.753374 s, 69.6 MB/s /mnt/lustre/f45.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 389 0x185 0x280000401 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873342 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.700072 s, 74.9 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.713273 s, 73.5 MB/s 52428800 bytes (52 MB) copied, 0.654148 s, 80.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.745443 s, 70.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.675392 s, 77.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.304929 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.786196 s, 66.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.775765 s, 67.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.795716 s, 65.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.7682 s, 68.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.783983 s, 66.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.74571 s, 70.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.788251 s, 66.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.812331 s, 64.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.818127 s, 64.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.814111 s, 64.4 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147828 s, 355 MB/s 52428800 bytes (52 MB) copied, 0.177388 s, 296 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151674 s, 346 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247972 s, 211 MB/s 52428800 bytes (52 MB) copied, 0.364174 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13896 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259942 s, 202 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.241052 s, 217 MB/s 52428800 bytes (52 MB) copied, 0.286328 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153474 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387862 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252966 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144374 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250974 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131241 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397764 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826124 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115088 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137729 s, 381 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.378658 s, 138 MB/s 52428800 bytes (52 MB) copied, 0.445156 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094524 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766559 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160584 s, 326 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900217 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753627 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.77433 s, 29.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.78694 s, 29.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.78014 s, 29.5 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.7939 s, 29.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.84525 s, 28.4 MB/s 52428800 bytes (52 MB) copied, 1.85018 s, 28.3 MB/s 52428800 bytes (52 MB) copied, 0.147328 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.74335 s, 30.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.7708 s, 29.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.86022 s, 28.2 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.75494 s, 29.9 MB/s 52428800 bytes (52 MB) copied, 1.75583 s, 29.9 MB/s 52428800 bytes (52 MB) copied, 1.86406 s, 28.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.85498 s, 28.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.89134 s, 27.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819396 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.90563 s, 27.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.89473 s, 27.7 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934182 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931041 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705687 s, 743 MB/s 52428800 bytes (52 MB) copied, 0.152588 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736278 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12997 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140677 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197624 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.324897 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.299609 s, 175 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.340111 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.220036 s, 238 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.296533 s, 177 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.320275 s, 164 MB/s 52428800 bytes (52 MB) copied, 0.146892 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.357487 s, 147 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352383 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367936 s, 142 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.317649 s, 165 MB/s 52428800 bytes (52 MB) copied, 0.244008 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.447731 s, 117 MB/s 52428800 bytes (52 MB) copied, 0.208883 s, 251 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.229329 s, 229 MB/s , 0.304061 s, 172 MB/s 52428800 bytes (52 MB) copied, 0.293644 s, 179 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202736 s, 259 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225936 s, 232 MB/s 52428800 bytes (52 MB) copied, 0.273408 s, 192 MB/s , 0.193212 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132439 s, 396 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267182 s, 196 MB/s 52428800 bytes (52 MB) copied, 0.249632 s, 210 MB/s 52428800 bytes (52 MB) copied, 0.19693 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135415 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.205965 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145568 s, 360 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247382 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135741 s, 386 MB/s 52428800 bytes (52 MB) copied, 0.209566 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212869 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422937 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620104 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037958 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645554 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727586 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100334 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708043 s, 740 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917457 s, 571 MB/s 52428800 bytes (52 MB) copied, 0.08446 s, 621 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104146 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116221 s, 451 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127239 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252889 s, 207 MB/s 52428800 bytes (52 MB) copied, 0.151714 s, 346 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229056 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215212 s, 244 MB/s 52428800 bytes (52 MB) copied, 0.202147 s, 259 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193259 s, 271 MB/s 52428800 bytes (52 MB) copied, 0.170644 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247697 s, 212 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231428 s, 227 MB/s 52428800 bytes (52 MB) copied, 0.242847 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247169 s, 212 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283219 s, 185 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245411 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217875 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260428 s, 201 MB/s 52428800 bytes (52 MB) copied, 0.368649 s, 142 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253053 s, 207 MB/s 52428800 bytes (52 MB) copied, 0.31607 s, 166 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127471 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131949 s, 397 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.499044 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.35156 s, 149 MB/s 52428800 bytes (52 MB) copied, 0.35433 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.338658 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387815 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140459 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33082 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191113 s, 274 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244062 s, 215 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.675543 s, 77.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237576 s, 221 MB/s 52428800 bytes (52 MB) copied, 0.246162 s, 213 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.304133 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.257113 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12583 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.303209 s, 173 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160696 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.285799 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189477 s, 277 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917966 s, 571 MB/s 52428800 bytes (52 MB) copied, 0.24961 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168309 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233176 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139232 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954347 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24309 s, 216 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139048 s, 377 MB/s 52428800 bytes (52 MB) copied, 0.214121 s, 245 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189397 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196346 s, 267 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167088 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126216 s, 415 MB/s 52428800 bytes (52 MB) copied, 0.0885474 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139132 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139938 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203384 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174413 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149312 s, 351 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187394 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160317 s, 327 MB/s 52428800 bytes (52 MB) copied, 0.150112 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159853 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138108 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200442 s, 262 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186545 s, 281 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207259 s, 253 MB/s 52428800 bytes (52 MB) copied, 0.198721 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149303 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248136 s, 211 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13602 s, 385 MB/s 52428800 bytes (52 MB) copied, 0.231081 s, 227 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176824 s, 297 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197668 s, 265 MB/s 52428800 bytes (52 MB) copied, 0.146068 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351312 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.299237 s, 175 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163229 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.307571 s, 170 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180557 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195191 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17404 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267703 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201183 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.28875 s, 182 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258232 s, 203 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204339 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225769 s, 232 MB/s 52428800 bytes (52 MB) copied, 0.350253 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248677 s, 211 MB/s 52428800 bytes (52 MB) copied, 0.23302 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.295263 s, 178 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.374411 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33275 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.311119 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.235976 s, 222 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239184 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213192 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20398 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121195 s, 433 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.302613 s, 173 MB/s 52428800 bytes (52 MB) copied, 0.197261 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175303 s, 299 MB/s 52428800 bytes (52 MB) copied, 0.327877 s, 160 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.371619 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159521 s, 329 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269586 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.319938 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103218 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166258 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132993 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890371 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471596 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093496 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949442 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161941 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.437102 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203843 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251643 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499163 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074939 s, 700 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153706 s, 341 MB/s 52428800 bytes (52 MB) copied, 0.201196 s, 261 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114246 s, 459 MB/s , 0.208585 s, 251 MB/s 52428800 bytes (52 MB) copied, 0.22861 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15443 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15615 s, 336 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202721 s, 259 MB/s 52428800 bytes (52 MB) copied, 0.222933 s, 235 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211393 s, 248 MB/s 52428800 bytes (52 MB) copied, 0.188783 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251157 s, 209 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22613 s, 232 MB/s 52428800 bytes (52 MB) copied, 0.224204 s, 234 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.221333 s, 237 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271056 s, 193 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.324849 s, 161 MB/s 52428800 bytes (52 MB) copied, 0.25719 s, 204 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242967 s, 216 MB/s 52428800 bytes (52 MB) copied, 0.35893 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260539 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205311 s, 255 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355189 s, 148 MB/s 52428800 bytes (52 MB) copied, 0.223345 s, 235 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.320516 s, 164 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123059 s, 426 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149192 s, 351 MB/s 52428800 bytes (52 MB) copied, 0.25669 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259837 s, 202 MB/s 52428800 bytes (52 MB) copied, 0.118028 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.254949 s, 206 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.321745 s, 163 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228564 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232358 s, 226 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803056 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184941 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103874 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154741 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270089 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701216 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141711 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924273 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641479 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399633 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671362 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528607 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172815 s, 303 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160249 s, 327 MB/s 52428800 bytes (52 MB) copied, 0.171799 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206454 s, 254 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150589 s, 348 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166039 s, 316 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342641 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.350928 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292495 s, 179 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189144 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.284624 s, 184 MB/s 52428800 bytes (52 MB) copied, 0.152568 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183603 s, 286 MB/s 52428800 bytes (52 MB) copied, 0.29139 s, 180 MB/s 52428800 bytes (52 MB) copied, 0.152269 s, 344 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.316151 s, 166 MB/s 52428800 bytes (52 MB) copied, 0.273715 s, 192 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282501 s, 186 MB/s 52428800 bytes (52 MB) copied, 0.331537 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.399568 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.327944 s, 160 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100486 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.365929 s, 143 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298953 s, 175 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143588 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.452834 s, 116 MB/s 52428800 bytes (52 MB) copied, 0.0944183 s, 555 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177038 s, 296 MB/s 52428800 bytes (52 MB) copied, 0.168046 s, 312 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247298 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158614 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252445 s, 208 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275263 s, 190 MB/s , 0.372889 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130828 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.322518 s, 163 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.226699 s, 231 MB/s , 0.239266 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584779 s, 897 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119148 s, 440 MB/s 52428800 bytes (52 MB) copied, 0.10348 s, 507 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252085 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192585 s, 272 MB/s 52428800 bytes (52 MB) copied, 0.279032 s, 188 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240998 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214842 s, 244 MB/s 52428800 bytes (52 MB) copied, 0.201167 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171449 s, 306 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.221262 s, 237 MB/s 52428800 bytes (52 MB) copied, 0.234919 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17528 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198259 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136161 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214264 s, 245 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193755 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230564 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19544 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14377 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630266 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143067 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108888 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125133 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943484 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0977075 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609887 s, 860 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13835 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732697 s, 716 MB/s 52428800 bytes (52 MB) copied, 0.133613 s, 392 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12571 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163993 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125158 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227643 s, 230 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275392 s, 190 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280108 s, 187 MB/s 52428800 bytes (52 MB) copied, 0.256715 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.312156 s, 168 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196723 s, 267 MB/s 52428800 bytes (52 MB) copied, 0.21914 s, 239 MB/s 52428800 bytes (52 MB) copied, 0.183283 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207774 s, 252 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.314804 s, 167 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.327728 s, 160 MB/s 52428800 bytes (52 MB) copied, 0.135112 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.403203 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210249 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177555 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137074 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212441 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.395666 s, 133 MB/s , 0.161765 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190005 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126691 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.478192 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21457 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367964 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198442 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237957 s, 220 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326646 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215064 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976669 s, 537 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898939 s, 583 MB/s 52428800 bytes (52 MB) copied, 0.107215 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252961 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236139 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190619 s, 275 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677926 s, 773 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111733 s, 469 MB/s 52428800 bytes (52 MB) copied, 0.114592 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068622 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755652 s, 694 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.147418 s, 356 MB/s , 0.0927446 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871719 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716808 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934361 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110089 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142929 s, 367 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102865 s, 510 MB/s 52428800 bytes (52 MB) copied, 0.154023 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129386 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152641 s, 343 MB/s 52428800 bytes (52 MB) copied, 0.155076 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166115 s, 316 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199836 s, 262 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147363 s, 356 MB/s 52428800 bytes (52 MB) copied, 0.12492 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172662 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182761 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111023 s, 472 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109758 s, 478 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186515 s, 281 MB/s 52428800 bytes (52 MB) copied, 0.0956733 s, 548 MB/s , 0.164975 s, 318 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10287 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116926 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145414 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118399 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153591 s, 341 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115428 s, 454 MB/s 52428800 bytes (52 MB) copied, 0.100736 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12021 s, 436 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181571 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145565 s, 360 MB/s 52428800 bytes (52 MB) copied, 0.145621 s, 360 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184953 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1533 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196567 s, 267 MB/s 52428800 bytes (52 MB) copied, 0.188625 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122206 s, 429 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162171 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124193 s, 422 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139506 s, 376 MB/s , 0.176574 s, 297 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123623 s, 424 MB/s 52428800 bytes (52 MB) copied, 0.106744 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216957 s, 242 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188963 s, 277 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248263 s, 211 MB/s 52428800 bytes (52 MB) copied, 0.212432 s, 247 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200609 s, 261 MB/s 52428800 bytes (52 MB) copied, 0.164317 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234725 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215573 s, 243 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259677 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.405052 s, 129 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227347 s, 231 MB/s 52428800 bytes (52 MB) copied, 0.270532 s, 194 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.172848 s, 303 MB/s , 0.226653 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197208 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.256214 s, 205 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237791 s, 220 MB/s 52428800 bytes (52 MB) copied, 0.196013 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253019 s, 207 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180141 s, 291 MB/s 52428800 bytes (52 MB) copied, 0.177647 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.418316 s, 125 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19351 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252564 s, 208 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.256358 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.325129 s, 161 MB/s 52428800 bytes (52 MB) copied, 0.139035 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352734 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190729 s, 275 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188153 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106832 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656289 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132675 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150252 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313126 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120732 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111541 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264967 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143172 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.26625 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.392826 s, 133 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232305 s, 226 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203063 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.317557 s, 165 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.375399 s, 140 MB/s 52428800 bytes (52 MB) copied, 0.253126 s, 207 MB/s 52428800 bytes (52 MB) copied, 0.316553 s, 166 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203512 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.462319 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.329184 s, 159 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.479298 s, 109 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.44015 s, 119 MB/s 52428800 bytes (52 MB) copied, 0.222396 s, 236 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543043 s, 96.5 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.260514 s, 201 MB/s , 0.408771 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.303073 s, 173 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.32062 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.503641 s, 104 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237791 s, 220 MB/s 52428800 bytes (52 MB) copied, 0.403142 s, 130 MB/s 52428800 bytes (52 MB) copied, 0.356493 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326977 s, 160 MB/s 52428800 bytes (52 MB) copied, 0.526579 s, 99.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.458936 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397827 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210715 s, 249 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200626 s, 261 MB/s 52428800 bytes (52 MB) copied, 0.0920002 s, 570 MB/s , 0.288433 s, 182 MB/s 52428800 bytes (52 MB) copied, 0.156682 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185576 s, 283 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251755 s, 208 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.12064 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153628 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.26011 s, 202 MB/s 52428800 bytes (52 MB) copied, 0.227047 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119541 s, 439 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265817 s, 197 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.257743 s, 203 MB/s 52428800 bytes (52 MB) copied, 0.262961 s, 199 MB/s 52428800 bytes (52 MB) copied, 0.167631 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290283 s, 181 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174201 s, 301 MB/s 52428800 bytes (52 MB) copied, 0.252395 s, 208 MB/s 52428800 bytes (52 MB) copied, 0.16443 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19954 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233638 s, 224 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204207 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270573 s, 194 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292365 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180949 s, 290 MB/s 52428800 bytes (52 MB) copied, 0.183888 s, 285 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124003 s, 423 MB/s 52428800 bytes (52 MB) copied, 0.203247 s, 258 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112162 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112253 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189219 s, 277 MB/s 52428800 bytes (52 MB) copied, 0.106868 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154592 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21863 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207161 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163289 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.189684 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146996 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156707 s, 335 MB/s 52428800 bytes (52 MB) copied, 0.197626 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153597 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199499 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147607 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262435 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20202 s, 260 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181119 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193515 s, 271 MB/s 52428800 bytes (52 MB) copied, 0.142704 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194304 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182208 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174132 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21066 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250828 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.404814 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.28389 s, 185 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215071 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24469 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266109 s, 197 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231072 s, 227 MB/s 52428800 bytes (52 MB) copied, 0.231294 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174676 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267025 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258752 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107824 s, 486 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.315648 s, 166 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.297264 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211848 s, 247 MB/s 52428800 bytes (52 MB) copied, 0.127973 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225421 s, 233 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212074 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138644 s, 378 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199922 s, 262 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.273287 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130818 s, 401 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125681 s, 417 MB/s , 0.102499 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100382 s, 522 MB/s 52428800 bytes (52 MB) copied, 0.219535 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07246 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157565 s, 333 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958983 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128987 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582337 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567857 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401928 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777334 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359439 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809629 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983562 s, 533 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124831 s, 420 MB/s 52428800 bytes (52 MB) copied, 0.109572 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137191 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129174 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134359 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.235343 s, 223 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227362 s, 231 MB/s 52428800 bytes (52 MB) copied, 0.128031 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131602 s, 398 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18557 s, 283 MB/s 52428800 bytes (52 MB) copied, 0.177486 s, 295 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179454 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159333 s, 329 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.203092 s, 258 MB/s , 0.183959 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10498 s, 499 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188509 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176493 s, 297 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219905 s, 238 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358372 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115501 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145659 s, 360 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161063 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126404 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17495 s, 300 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100075 s, 524 MB/s 52428800 bytes (52 MB) copied, 0.105469 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969691 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879249 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139225 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437139 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529573 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549619 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479456 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673084 s, 779 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119356 s, 439 MB/s 52428800 bytes (52 MB) copied, 0.113732 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136422 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100633 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0834478 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119022 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929621 s, 564 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199554 s, 263 MB/s 52428800 bytes (52 MB) copied, 0.149228 s, 351 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161093 s, 325 MB/s 52428800 bytes (52 MB) copied, 0.164084 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14921 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194017 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107062 s, 490 MB/s 52428800 bytes (52 MB) copied, 0.141089 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164382 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166419 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155118 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202749 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199442 s, 263 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168531 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.235389 s, 223 MB/s 52428800 bytes (52 MB) copied, 0.144648 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129101 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230276 s, 228 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135663 s, 386 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133294 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169097 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154538 s, 339 MB/s 52428800 bytes (52 MB) copied, 0.154009 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0808707 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129283 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12658 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139794 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152923 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114944 s, 456 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0957388 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15343 s, 342 MB/s 52428800 bytes (52 MB) copied, 0.138086 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114842 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108153 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148752 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114697 s, 457 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167276 s, 313 MB/s , 0.143448 s, 365 MB/s 52428800 bytes (52 MB) copied, 0.100951 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15136 s, 346 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226127 s, 232 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0884096 s, 593 MB/s 52428800 bytes (52 MB) copied, 0.171667 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171327 s, 306 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249408 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.302223 s, 173 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19448 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21419 s, 245 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222803 s, 235 MB/s 52428800 bytes (52 MB) copied, 0.228623 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176523 s, 297 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159908 s, 328 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326304 s, 161 MB/s 52428800 bytes (52 MB) copied, 0.165567 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144641 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129708 s, 404 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211045 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.285201 s, 184 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182768 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868219 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195887 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.28692 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253667 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.276791 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208591 s, 251 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190973 s, 275 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134884 s, 389 MB/s 52428800 bytes (52 MB) copied, 0.20867 s, 251 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230968 s, 227 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213413 s, 246 MB/s 52428800 bytes (52 MB) copied, 0.201857 s, 260 MB/s 52428800 bytes (52 MB) copied, 0.227159 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156598 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126144 s, 416 MB/s , 0.140355 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280551 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559858 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061349 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173435 s, 302 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151824 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167215 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136294 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166467 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208189 s, 252 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0834788 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681382 s, 769 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146172 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154618 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947057 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187665 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109133 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815652 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754632 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794413 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499061 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177431 s, 295 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901841 s, 581 MB/s 52428800 bytes (52 MB) copied, 0.10934 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128635 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631901 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438342 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10576 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217934 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202059 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379805 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090887 s, 577 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209755 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188254 s, 279 MB/s 52428800 bytes (52 MB) copied, 0.10044 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104153 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706209 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525331 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130754 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760804 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111906 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117059 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103196 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489689 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.086981 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0939521 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605505 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084585 s, 620 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800332 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108113 s, 485 MB/s 52428800 bytes (52 MB) copied, 0.0930724 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141173 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147968 s, 354 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156237 s, 336 MB/s 52428800 bytes (52 MB) copied, 0.106827 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150907 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114382 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11988 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161155 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826796 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213495 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142563 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130296 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186214 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16762 s, 313 MB/s , 0.178512 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181248 s, 289 MB/s , 0.212417 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291809 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24728 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143574 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129731 s, 404 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173571 s, 302 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18198 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209455 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185845 s, 282 MB/s 52428800 bytes (52 MB) copied, 0.153732 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13492 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211054 s, 248 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108914 s, 481 MB/s 52428800 bytes (52 MB) copied, 0.184086 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175603 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175767 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134093 s, 391 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135442 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101387 s, 517 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0878485 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106996 s, 490 MB/s 52428800 bytes (52 MB) copied, 0.090855 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931665 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740195 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145902 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768293 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199178 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753993 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11162 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114441 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0963557 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103656 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543735 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129244 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349806 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396126 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795577 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396394 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310575 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045256 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662882 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105726 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535677 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114089 s, 460 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163567 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0596114 s, 880 MB/s 52428800 bytes (52 MB) copied, 0.0655698 s, 800 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186862 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108766 s, 482 MB/s 52428800 bytes (52 MB) copied, 0.108684 s, 482 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.302744 s, 173 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.411099 s, 128 MB/s 52428800 bytes (52 MB) copied, 0.214004 s, 245 MB/s 52428800 bytes (52 MB) copied, 0.268471 s, 195 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.309119 s, 170 MB/s 52428800 bytes (52 MB) copied, 0.261647 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243577 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2103 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.295796 s, 177 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251164 s, 209 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.328481 s, 160 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265422 s, 198 MB/s 52428800 bytes (52 MB) copied, 0.263805 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.38266 s, 137 MB/s 52428800 bytes (52 MB) copied, 0.274023 s, 191 MB/s 52428800 bytes (52 MB) copied, 0.218404 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.394676 s, 133 MB/s 52428800 bytes (52 MB) copied, 0.157688 s, 332 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269239 s, 195 MB/s 52428800 bytes (52 MB) copied, 0.267154 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410078 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.365396 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298066 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231917 s, 226 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227699 s, 230 MB/s 52428800 bytes (52 MB) copied, 0.480509 s, 109 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145874 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157396 s, 333 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670142 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132222 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175814 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125093 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459048 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031061 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304186 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444188 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0272128 s, 1.9 GB/s , 0.0376438 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629457 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559503 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444139 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703254 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122716 s, 427 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.132131 s, 397 MB/s , 0.123828 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102233 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17092 s, 307 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14426 s, 363 MB/s 52428800 bytes (52 MB) copied, 0.138564 s, 378 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135995 s, 386 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904577 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136231 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112247 s, 467 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098834 s, 530 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0895096 s, 586 MB/s 52428800 bytes (52 MB) copied, 0.127023 s, 413 MB/s 52428800 bytes (52 MB) copied, 0.118168 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514679 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402514 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102379 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.096914 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743523 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123978 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131869 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121494 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132981 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12002 s, 437 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116189 s, 451 MB/s 52428800 bytes (52 MB) copied, 0.119941 s, 437 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113109 s, 464 MB/s 52428800 bytes (52 MB) copied, 0.122217 s, 429 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.172187 s, 304 MB/s , 0.0709199 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117636 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19408 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174451 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203398 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785542 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170457 s, 308 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0582262 s, 900 MB/s , 0.079936 s, 656 MB/s 52428800 bytes (52 MB) copied, 0.128552 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819073 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113845 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093625 s, 560 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617099 s, 850 MB/s 52428800 bytes (52 MB) copied, 0.133148 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822534 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753296 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740399 s, 708 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771674 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997407 s, 526 MB/s 52428800 bytes (52 MB) copied, 0.139808 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144139 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150845 s, 348 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141217 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232628 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183116 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148761 s, 352 MB/s 52428800 bytes (52 MB) copied, 0.191126 s, 274 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154357 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184345 s, 284 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.1841 s, 285 MB/s , 0.171557 s, 306 MB/s , 0.15191 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152677 s, 343 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17312 s, 303 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.113254 s, 463 MB/s 52428800 bytes (52 MB) copied, 0.149479 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170884 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16859 s, 311 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157982 s, 332 MB/s 52428800 bytes (52 MB) copied, 0.141569 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850913 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184372 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135721 s, 386 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242602 s, 216 MB/s 52428800 bytes (52 MB) copied, 0.123727 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111863 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157417 s, 333 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175816 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131145 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157123 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258998 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165828 s, 316 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339579 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194402 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203849 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17673 s, 297 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244171 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131135 s, 400 MB/s 52428800 bytes (52 MB) copied, 0.153465 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14423 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210621 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219163 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237808 s, 220 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399405 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194451 s, 270 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.284331 s, 184 MB/s 52428800 bytes (52 MB) copied, 0.123431 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23808 s, 220 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123542 s, 424 MB/s 52428800 bytes (52 MB) copied, 0.135808 s, 386 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.345033 s, 152 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243931 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168627 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207267 s, 253 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127608 s, 411 MB/s , 0.386654 s, 136 MB/s 52428800 bytes (52 MB) copied, 0.124051 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0916253 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182842 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189818 s, 276 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.423205 s, 124 MB/s 52428800 bytes (52 MB) copied, 0.174081 s, 301 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14528 s, 361 MB/s , 0.163462 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126203 s, 415 MB/s 52428800 bytes (52 MB) copied, 0.143513 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569408 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121531 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813073 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479002 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675108 s, 777 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555497 s, 944 MB/s 52428800 bytes (52 MB) copied, 0.100667 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498225 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134284 s, 390 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114177 s, 459 MB/s 52428800 bytes (52 MB) copied, 0.130067 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10449 s, 502 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725374 s, 723 MB/s 52428800 bytes (52 MB) copied, 0.0549729 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132972 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742161 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122206 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646269 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528485 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382527 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043912 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.107419 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904032 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670344 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11047 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161133 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200238 s, 262 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171321 s, 306 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283908 s, 185 MB/s 52428800 bytes (52 MB) copied, 0.236817 s, 221 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136222 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226479 s, 231 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214937 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23959 s, 219 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244362 s, 215 MB/s 52428800 bytes (52 MB) copied, 0.21166 s, 248 MB/s 52428800 bytes (52 MB) copied, 0.278654 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172295 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242902 s, 216 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191364 s, 274 MB/s 52428800 bytes (52 MB) copied, 0.184296 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166297 s, 315 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271321 s, 193 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110604 s, 474 MB/s 52428800 bytes (52 MB) copied, 0.165604 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20198 s, 260 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217864 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246823 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162136 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.297104 s, 176 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.286178 s, 183 MB/s 52428800 bytes (52 MB) copied, 0.20264 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147066 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263605 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152757 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931027 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141438 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436278 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.236681 s, 222 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237354 s, 221 MB/s 52428800 bytes (52 MB) copied, 0.126531 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136976 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156238 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107263 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140879 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.223677 s, 234 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1913 s, 274 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142218 s, 369 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179894 s, 291 MB/s 52428800 bytes (52 MB) copied, 0.248584 s, 211 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242673 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16465 s, 318 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219812 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171075 s, 306 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282363 s, 186 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.126554 s, 414 MB/s , 0.148669 s, 353 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224491 s, 234 MB/s 52428800 bytes (52 MB) copied, 0.246551 s, 213 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215376 s, 243 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156732 s, 335 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219201 s, 239 MB/s 52428800 bytes (52 MB) copied, 0.138915 s, 377 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150045 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151846 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.29928 s, 175 MB/s 52428800 bytes (52 MB) copied, 0.280585 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.255196 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1341 s, 391 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239329 s, 219 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159382 s, 329 MB/s 52428800 bytes (52 MB) copied, 0.152873 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239951 s, 218 MB/s 52428800 bytes (52 MB) copied, 0.153662 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185854 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127913 s, 410 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173293 s, 303 MB/s 52428800 bytes (52 MB) copied, 0.171342 s, 306 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18561 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335892 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171185 s, 306 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195243 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269528 s, 195 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129403 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275065 s, 191 MB/s 52428800 bytes (52 MB) copied, 0.331287 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145703 s, 360 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167762 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192357 s, 273 MB/s 52428800 bytes (52 MB) copied, 0.133983 s, 391 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156604 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142217 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116455 s, 450 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100562 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149047 s, 352 MB/s 52428800 bytes (52 MB) copied, 0.132592 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126507 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761977 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126594 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865008 s, 606 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148098 s, 354 MB/s 52428800 bytes (52 MB) copied, 0.119469 s, 439 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152654 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872243 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117467 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178769 s, 293 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160354 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161329 s, 325 MB/s , 0.134696 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155258 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152557 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157109 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164812 s, 318 MB/s , 0.173191 s, 303 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167888 s, 312 MB/s 52428800 bytes (52 MB) copied, 0.160213 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174496 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168323 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117889 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175923 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694107 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24845 s, 211 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238175 s, 220 MB/s 52428800 bytes (52 MB) copied, 0.237708 s, 221 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146536 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206635 s, 254 MB/s 52428800 bytes (52 MB) copied, 0.2998 s, 175 MB/s 52428800 bytes (52 MB) copied, 0.272812 s, 192 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191635 s, 274 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.124781 s, 420 MB/s , 0.167828 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121827 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.276788 s, 189 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139507 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157584 s, 333 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244767 s, 214 MB/s 52428800 bytes (52 MB) copied, 0.0813212 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154279 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270001 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.293144 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.334525 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.429097 s, 122 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.285598 s, 184 MB/s , 0.0770206 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279626 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133427 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169883 s, 309 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0860689 s, 609 MB/s 52428800 bytes (52 MB) copied, 0.122481 s, 428 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167243 s, 313 MB/s 52428800 bytes (52 MB) copied, 0.060277 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199687 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406991 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460459 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140106 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881339 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394311 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509778 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085977 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818352 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731297 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123595 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113696 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909718 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135681 s, 386 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322933 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112859 s, 465 MB/s 52428800 bytes (52 MB) copied, 0.119671 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13607 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575656 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706654 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624723 s, 839 MB/s , 0.120574 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882532 s, 594 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0908695 s, 577 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.14526 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132862 s, 395 MB/s 52428800 bytes (52 MB) copied, 0.108003 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108791 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182118 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125938 s, 416 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132483 s, 396 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173108 s, 303 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156558 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169033 s, 310 MB/s 52428800 bytes (52 MB) copied, 0.156862 s, 334 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.234835 s, 223 MB/s , 0.118745 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094782 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187561 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187109 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121526 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198795 s, 264 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103129 s, 508 MB/s 52428800 bytes (52 MB) copied, 0.110022 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172677 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184822 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158297 s, 331 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459666 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523283 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0453192 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983312 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037197 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085416 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069988 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104632 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111304 s, 471 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910022 s, 576 MB/s 52428800 bytes (52 MB) copied, 0.122432 s, 428 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125976 s, 416 MB/s 52428800 bytes (52 MB) copied, 0.0733122 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144177 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132036 s, 397 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132599 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164393 s, 319 MB/s 52428800 bytes (52 MB) copied, 0.150553 s, 348 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138394 s, 379 MB/s , 0.190734 s, 275 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.131456 s, 399 MB/s , 0.138479 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252339 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270659 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.27346 s, 192 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124121 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14327 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348627 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.332029 s, 158 MB/s 52428800 bytes (52 MB) copied, 0.153009 s, 343 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.303641 s, 173 MB/s 52428800 bytes (52 MB) copied, 0.318484 s, 165 MB/s 52428800 bytes (52 MB) copied, 0.234489 s, 224 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.370947 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212501 s, 247 MB/s 52428800 bytes (52 MB) copied, 0.169254 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244071 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126676 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186806 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139497 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110121 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.309603 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21501 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226403 s, 232 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121674 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140733 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355655 s, 147 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145408 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.313051 s, 167 MB/s 52428800 bytes (52 MB) copied, 0.187441 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102953 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218504 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105803 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142548 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126155 s, 416 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275158 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136775 s, 383 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131747 s, 398 MB/s 52428800 bytes (52 MB) copied, 0.156571 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119322 s, 439 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081083 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150227 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0921572 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252345 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186806 s, 281 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179181 s, 293 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233891 s, 224 MB/s , 0.162047 s, 324 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258801 s, 203 MB/s 52428800 bytes (52 MB) copied, 0.180672 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173208 s, 303 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213377 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204093 s, 257 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211691 s, 248 MB/s 52428800 bytes (52 MB) copied, 0.183372 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151424 s, 346 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0960348 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167011 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.254891 s, 206 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1284 s, 408 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154249 s, 340 MB/s 52428800 bytes (52 MB) copied, 0.137156 s, 382 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0973284 s, 539 MB/s 52428800 bytes (52 MB) copied, 0.145031 s, 362 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118029 s, 444 MB/s 52428800 bytes (52 MB) copied, 0.193451 s, 271 MB/s 52428800 bytes (52 MB) copied, 0.195468 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213316 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137061 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138336 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164147 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177044 s, 296 MB/s , 0.133421 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163969 s, 320 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655978 s, 799 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.129703 s, 404 MB/s 52428800 bytes (52 MB) copied, 0.102127 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953325 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812567 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130755 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943849 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.131675 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785268 s, 668 MB/s 52428800 bytes (52 MB) copied, 0.163865 s, 320 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125635 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.145474 s, 360 MB/s , 0.123245 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147058 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111861 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599675 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202312 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177394 s, 296 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151181 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133357 s, 393 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.136689 s, 384 MB/s , 0.187136 s, 280 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159111 s, 330 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12429 s, 422 MB/s 52428800 bytes (52 MB) copied, 0.147276 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078661 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103557 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668602 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515452 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632226 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449964 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269405 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686882 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678716 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427023 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027272 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402177 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667525 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396499 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534448 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471825 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360534 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570463 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705868 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655195 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362822 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845424 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109528 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146928 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103798 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177528 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131074 s, 400 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0938078 s, 559 MB/s 52428800 bytes (52 MB) copied, 0.128384 s, 408 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127564 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15811 s, 332 MB/s 52428800 bytes (52 MB) copied, 0.12843 s, 408 MB/s 52428800 bytes (52 MB) copied, 0.134285 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764807 s, 686 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154424 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113861 s, 460 MB/s 52428800 bytes (52 MB) copied, 0.147769 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147623 s, 355 MB/s , 0.146147 s, 359 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.11956 s, 439 MB/s 52428800 bytes (52 MB) copied, 0.12522 s, 419 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154234 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.148684 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845073 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107116 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122215 s, 429 MB/s , 0.0799014 s, 656 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147615 s, 355 MB/s 52428800 bytes (52 MB) copied, 0.0996648 s, 526 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102206 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793244 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176968 s, 296 MB/s 52428800 bytes (52 MB) copied, 0.100364 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102958 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154794 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430613 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932026 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446349 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589591 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555509 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038837 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891721 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778092 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106991 s, 490 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0916667 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192471 s, 272 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144611 s, 363 MB/s 52428800 bytes (52 MB) copied, 0.128631 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158486 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230136 s, 228 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266063 s, 197 MB/s 52428800 bytes (52 MB) copied, 0.316891 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278175 s, 188 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197239 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216308 s, 242 MB/s 52428800 bytes (52 MB) copied, 0.298774 s, 175 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.341852 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20332 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218425 s, 240 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.258663 s, 203 MB/s , 0.184509 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.31102 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.221106 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.364115 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.382397 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200615 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218895 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233602 s, 224 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348287 s, 151 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159147 s, 329 MB/s 52428800 bytes (52 MB) copied, 0.229354 s, 229 MB/s , 0.217639 s, 241 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149735 s, 350 MB/s 52428800 bytes (52 MB) copied, 0.191847 s, 273 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974276 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136515 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151686 s, 346 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037052 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342731 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0902424 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600418 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982317 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876235 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112158 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10147 s, 517 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105028 s, 499 MB/s 52428800 bytes (52 MB) copied, 0.0976333 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941636 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136488 s, 384 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192754 s, 272 MB/s 52428800 bytes (52 MB) copied, 0.0778709 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173032 s, 303 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172096 s, 305 MB/s 52428800 bytes (52 MB) copied, 0.148573 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162271 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155272 s, 338 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184237 s, 285 MB/s 52428800 bytes (52 MB) copied, 0.17229 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113836 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125278 s, 418 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161373 s, 325 MB/s 52428800 bytes (52 MB) copied, 0.104785 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179893 s, 291 MB/s 52428800 bytes (52 MB) copied, 0.160189 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885408 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116949 s, 448 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661579 s, 792 MB/s 52428800 bytes (52 MB) copied, 0.147505 s, 355 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139862 s, 375 MB/s 52428800 bytes (52 MB) copied, 0.0917855 s, 571 MB/s 52428800 bytes (52 MB) copied, 0.104426 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141456 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0988296 s, 530 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898463 s, 584 MB/s 52428800 bytes (52 MB) copied, 0.204509 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207775 s, 252 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0856998 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082277 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952719 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653062 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721765 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06313 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997518 s, 526 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0513434 s, 1.0 GB/s , 0.125772 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07115 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761197 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125723 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112279 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345962 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680086 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380584 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592968 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041189 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756782 s, 693 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310185 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.130272 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045473 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401408 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538257 s, 974 MB/s 52428800 bytes (52 MB) copied, 0.0493501 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941147 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103167 s, 508 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850384 s, 617 MB/s 52428800 bytes (52 MB) copied, 0.120724 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110556 s, 474 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121447 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115847 s, 453 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.126288 s, 415 MB/s , 0.120351 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125032 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122706 s, 427 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170076 s, 308 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180145 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270222 s, 194 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161844 s, 324 MB/s , 0.154169 s, 340 MB/s 52428800 bytes (52 MB) copied, 0.190069 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180063 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187795 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200898 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247519 s, 212 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.344663 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176543 s, 297 MB/s 52428800 bytes (52 MB) copied, 0.162727 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.254202 s, 206 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249422 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217479 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.268447 s, 195 MB/s 52428800 bytes (52 MB) copied, 0.275263 s, 190 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213141 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290526 s, 180 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249975 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134758 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145182 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.362968 s, 144 MB/s 52428800 bytes (52 MB) copied, 0.131179 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246707 s, 213 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164978 s, 318 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233998 s, 224 MB/s 52428800 bytes (52 MB) copied, 0.244993 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0990801 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182416 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055078 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285994 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101236 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065431 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893178 s, 587 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981045 s, 534 MB/s 52428800 bytes (52 MB) copied, 0.0880751 s, 595 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0896884 s, 585 MB/s 52428800 bytes (52 MB) copied, 0.132608 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144028 s, 364 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124702 s, 420 MB/s 52428800 bytes (52 MB) copied, 0.105569 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162924 s, 322 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148147 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167236 s, 314 MB/s 52428800 bytes (52 MB) copied, 0.160109 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127747 s, 410 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879892 s, 596 MB/s 52428800 bytes (52 MB) copied, 0.126374 s, 415 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196604 s, 267 MB/s , 0.135391 s, 387 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130476 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100313 s, 523 MB/s 52428800 bytes (52 MB) copied, 0.164848 s, 318 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.103682 s, 506 MB/s , 0.112772 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543232 s, 965 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.100873 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109027 s, 481 MB/s , 0.1214 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108001 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445937 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670782 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673031 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376618 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095008 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101321 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102077 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501814 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027705 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146843 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180471 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744054 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123381 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646578 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119554 s, 439 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626638 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779103 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102013 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065437 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145157 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123118 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954952 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681252 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506998 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305453 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100391 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302084 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100948 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057529 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906148 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100951 s, 519 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702976 s, 746 MB/s 52428800 bytes (52 MB) copied, 0.0949798 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446647 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730516 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100632 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113134 s, 463 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134166 s, 391 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152301 s, 344 MB/s 52428800 bytes (52 MB) copied, 0.114725 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100957 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943274 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146087 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165648 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163482 s, 321 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206684 s, 254 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238646 s, 220 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166243 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270294 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200997 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214777 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166169 s, 316 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259816 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216319 s, 242 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280676 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263884 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.211936 s, 247 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out , 0.165964 s, 316 MB/s , 0.356244 s, 147 MB/s 52428800 bytes (52 MB) copied, 0.204324 s, 257 MB/s 52428800 bytes (52 MB) copied, 0.181183 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139242 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1784 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225492 s, 233 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.293615 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247441 s, 212 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120187 s, 436 MB/s 52428800 bytes (52 MB) copied, 0.219304 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775737 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151032 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214719 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711196 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684512 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181304 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559477 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399123 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440295 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303783 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839551 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566025 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066292 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0984834 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919342 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820537 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549147 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101479 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497249 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671799 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117288 s, 447 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651327 s, 805 MB/s 52428800 bytes (52 MB) copied, 0.0479636 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812763 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563461 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152308 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132094 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171896 s, 305 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148191 s, 354 MB/s 52428800 bytes (52 MB) copied, 0.13037 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178438 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176346 s, 297 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.221883 s, 236 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237371 s, 221 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206824 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277183 s, 189 MB/s , 0.175355 s, 299 MB/s 52428800 bytes (52 MB) copied, 0.268988 s, 195 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154355 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283925 s, 185 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148004 s, 354 MB/s 52428800 bytes (52 MB) copied, 0.190676 s, 275 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145781 s, 360 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292657 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18061 s, 290 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.316579 s, 166 MB/s 52428800 bytes (52 MB) copied, 0.180373 s, 291 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259951 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.286544 s, 183 MB/s 52428800 bytes (52 MB) copied, 0.249281 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118157 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265678 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100566 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218157 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185218 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21957 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213881 s, 245 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142061 s, 369 MB/s 52428800 bytes (52 MB) copied, 0.121622 s, 431 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152131 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100083 s, 524 MB/s 52428800 bytes (52 MB) copied, 0.128782 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169032 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153362 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188758 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652852 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697522 s, 752 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204482 s, 256 MB/s 52428800 bytes (52 MB) copied, 0.159427 s, 329 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164258 s, 319 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111972 s, 468 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125945 s, 416 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.119564 s, 439 MB/s , 0.121647 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773341 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0867797 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259847 s, 202 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100074 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122646 s, 427 MB/s 52428800 bytes (52 MB) copied, 0.183952 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0915085 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140859 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179826 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137991 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133818 s, 392 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168481 s, 311 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147413 s, 356 MB/s 52428800 bytes (52 MB) copied, 0.171682 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164583 s, 319 MB/s , 0.14104 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14023 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163751 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0866486 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192043 s, 273 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213292 s, 246 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186116 s, 282 MB/s 52428800 bytes (52 MB) copied, 0.156566 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178824 s, 293 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.261705 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1331 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158245 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335839 s, 156 MB/s 50+0 records in 50+0 records out , 0.130738 s, 401 MB/s 52428800 bytes (52 MB) copied, 0.209273 s, 251 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.272224 s, 193 MB/s 52428800 bytes (52 MB) copied, 0.174827 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.300461 s, 174 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142875 s, 367 MB/s 52428800 bytes (52 MB) copied, 0.153703 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159711 s, 328 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103493 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15011 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266007 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182088 s, 288 MB/s 52428800 bytes (52 MB) copied, 0.16165 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24877 s, 211 MB/s 52428800 bytes (52 MB) copied, 0.22564 s, 232 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13321 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110149 s, 476 MB/s 52428800 bytes (52 MB) copied, 0.336375 s, 156 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271207 s, 193 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180952 s, 290 MB/s 52428800 bytes (52 MB) copied, 0.100342 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985838 s, 532 MB/s 52428800 bytes (52 MB) copied, 0.123488 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604328 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110109 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540268 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512134 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794346 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0913477 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049413 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0844772 s, 621 MB/s 52428800 bytes (52 MB) copied, 0.0895197 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732848 s, 715 MB/s , 0.0807083 s, 650 MB/s 52428800 bytes (52 MB) copied, 0.0622472 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933058 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393672 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290931 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313053 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503296 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.068243 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504873 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100553 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555535 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961754 s, 545 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983481 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135104 s, 388 MB/s 52428800 bytes (52 MB) copied, 0.163789 s, 320 MB/s 52428800 bytes (52 MB) copied, 0.129099 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120926 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152623 s, 344 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167566 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0940034 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174664 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126842 s, 413 MB/s , 0.182279 s, 288 MB/s 52428800 bytes (52 MB) copied, 0.141572 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.223135 s, 235 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226058 s, 232 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197613 s, 265 MB/s 52428800 bytes (52 MB) copied, 0.174091 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193553 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17241 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2405 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207034 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245145 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217331 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175565 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251658 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250329 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212525 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180975 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188168 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.304038 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194969 s, 269 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184141 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292736 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19068 s, 275 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23096 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282389 s, 186 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1796 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278176 s, 188 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172992 s, 303 MB/s 52428800 bytes (52 MB) copied, 0.0932392 s, 562 MB/s 52428800 bytes (52 MB) copied, 0.182096 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195372 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069174 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10675 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135119 s, 388 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776346 s, 675 MB/s 52428800 bytes (52 MB) copied, 0.0878848 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05668 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437424 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266889 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308225 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815306 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728888 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111332 s, 471 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132191 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118145 s, 444 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135686 s, 386 MB/s 52428800 bytes (52 MB) copied, 0.0969366 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134695 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197 s, 266 MB/s , 0.149837 s, 350 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188848 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137498 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152105 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181304 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152942 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112384 s, 467 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178208 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160362 s, 327 MB/s 52428800 bytes (52 MB) copied, 0.189263 s, 277 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162838 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185516 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127437 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.254469 s, 206 MB/s , 0.130076 s, 403 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135668 s, 386 MB/s 52428800 bytes (52 MB) copied, 0.140552 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282536 s, 186 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182056 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266599 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247398 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188172 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144234 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162286 s, 323 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326542 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.274977 s, 191 MB/s , 0.329161 s, 159 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253257 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290875 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.331464 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.334272 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.344671 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.313353 s, 167 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263013 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263068 s, 199 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.337423 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.306189 s, 171 MB/s 52428800 bytes (52 MB) copied, 0.207289 s, 253 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.295758 s, 177 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.268281 s, 195 MB/s , 0.297096 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367828 s, 143 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156727 s, 335 MB/s 52428800 bytes (52 MB) copied, 0.168966 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291069 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226833 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218018 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186023 s, 282 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342622 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765903 s, 685 MB/s 52428800 bytes (52 MB) copied, 0.127273 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171289 s, 306 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11452 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277963 s, 189 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641682 s, 817 MB/s 52428800 bytes (52 MB) copied, 0.117571 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148926 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219425 s, 239 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142436 s, 368 MB/s 52428800 bytes (52 MB) copied, 0.140469 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0926417 s, 566 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157769 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136824 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14532 s, 361 MB/s 52428800 bytes (52 MB) copied, 0.153196 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101141 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198446 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18717 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138464 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16459 s, 319 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1333 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19321 s, 271 MB/s 52428800 bytes (52 MB) copied, 0.161713 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17894 s, 293 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118029 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159304 s, 329 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173229 s, 303 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0993827 s, 528 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179774 s, 292 MB/s 52428800 bytes (52 MB) copied, 0.289095 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147375 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17851 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184673 s, 284 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180212 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17068 s, 307 MB/s 52428800 bytes (52 MB) copied, 0.151584 s, 346 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191131 s, 274 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.236814 s, 221 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170626 s, 307 MB/s 52428800 bytes (52 MB) copied, 0.153332 s, 342 MB/s 52428800 bytes (52 MB) copied, 0.215424 s, 243 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227861 s, 230 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237243 s, 221 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188942 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244774 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205956 s, 255 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193021 s, 272 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.236059 s, 222 MB/s 52428800 bytes (52 MB) copied, 0.217528 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242898 s, 216 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266573 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189084 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198877 s, 264 MB/s 52428800 bytes (52 MB) copied, 0.25128 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271486 s, 193 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.30584 s, 171 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.353535 s, 148 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123889 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12641 s, 415 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251764 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266217 s, 197 MB/s 52428800 bytes (52 MB) copied, 0.25285 s, 207 MB/s 52428800 bytes (52 MB) copied, 0.187474 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136159 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.320137 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.388482 s, 135 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251284 s, 209 MB/s 52428800 bytes (52 MB) copied, 0.243284 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205796 s, 255 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168273 s, 312 MB/s 52428800 bytes (52 MB) copied, 0.182826 s, 287 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155716 s, 337 MB/s 52428800 bytes (52 MB) copied, 0.203124 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229384 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174602 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141162 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168272 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247002 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215734 s, 243 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204157 s, 257 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188405 s, 278 MB/s 52428800 bytes (52 MB) copied, 0.152751 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187173 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125143 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.403172 s, 130 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222266 s, 236 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224733 s, 233 MB/s 52428800 bytes (52 MB) copied, 0.118511 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.295793 s, 177 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230622 s, 227 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207345 s, 253 MB/s 52428800 bytes (52 MB) copied, 0.22953 s, 228 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.201908 s, 260 MB/s , 0.225115 s, 233 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727777 s, 720 MB/s 52428800 bytes (52 MB) copied, 0.226575 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124561 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948235 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0916506 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186643 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127632 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799397 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612101 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774499 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387836 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557209 s, 941 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333303 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0690151 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110903 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845163 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536234 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10622 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0963179 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103404 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154018 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20055 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214861 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339936 s, 154 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.286036 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.337561 s, 155 MB/s 52428800 bytes (52 MB) copied, 0.258852 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.310827 s, 169 MB/s , 0.376622 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.282076 s, 186 MB/s 52428800 bytes (52 MB) copied, 0.23618 s, 222 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283575 s, 185 MB/s 52428800 bytes (52 MB) copied, 0.396692 s, 132 MB/s 52428800 bytes (52 MB) copied, 0.252464 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.257641 s, 203 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.361302 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.394297 s, 133 MB/s , 0.253549 s, 207 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.4906 s, 107 MB/s 52428800 bytes (52 MB) copied, 0.352143 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.384419 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.268551 s, 195 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234731 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.471897 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198451 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.297242 s, 176 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173588 s, 302 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.319789 s, 164 MB/s 52428800 bytes (52 MB) copied, 0.2175 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159617 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.319244 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923478 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907791 s, 578 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143558 s, 365 MB/s 52428800 bytes (52 MB) copied, 0.209945 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206041 s, 254 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141614 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694567 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125462 s, 418 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19174 s, 273 MB/s 52428800 bytes (52 MB) copied, 0.179819 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151933 s, 345 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202728 s, 259 MB/s 52428800 bytes (52 MB) copied, 0.363753 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215489 s, 243 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166696 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186197 s, 282 MB/s 52428800 bytes (52 MB) copied, 0.255143 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203702 s, 257 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.2215 s, 237 MB/s , 0.240902 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.305919 s, 171 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171551 s, 306 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203926 s, 257 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149771 s, 350 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142766 s, 367 MB/s 52428800 bytes (52 MB) copied, 0.128558 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179962 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.31921 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159035 s, 330 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200778 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.465623 s, 113 MB/s 52428800 bytes (52 MB) copied, 0.361186 s, 145 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253116 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139883 s, 375 MB/s 52428800 bytes (52 MB) copied, 0.229322 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12356 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131153 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0960277 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0991344 s, 529 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173735 s, 302 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140751 s, 372 MB/s 52428800 bytes (52 MB) copied, 0.128818 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105554 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071388 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806091 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872332 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813195 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628688 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877337 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746364 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691179 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669007 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932284 s, 562 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234155 s, 224 MB/s 52428800 bytes (52 MB) copied, 0.225114 s, 233 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234768 s, 223 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250092 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265718 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269614 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.324712 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.350005 s, 150 MB/s 52428800 bytes (52 MB) copied, 0.289167 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202937 s, 258 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.276394 s, 190 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.285042 s, 184 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291095 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.274709 s, 191 MB/s 52428800 bytes (52 MB) copied, 0.147674 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.327231 s, 160 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.30773 s, 170 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318722 s, 164 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189223 s, 277 MB/s 52428800 bytes (52 MB) copied, 0.191578 s, 274 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.278945 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239022 s, 219 MB/s 52428800 bytes (52 MB) copied, 0.223381 s, 235 MB/s , 0.286444 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.373585 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278376 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147732 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176916 s, 296 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249861 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.241874 s, 217 MB/s , 0.0821721 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238376 s, 220 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.311853 s, 168 MB/s 52428800 bytes (52 MB) copied, 0.181094 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120278 s, 436 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210594 s, 249 MB/s 52428800 bytes (52 MB) copied, 0.104657 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0988515 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100688 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709575 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690849 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156106 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125344 s, 418 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148374 s, 353 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201623 s, 260 MB/s 52428800 bytes (52 MB) copied, 0.200577 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194561 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229112 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245243 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229215 s, 229 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.272657 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.336596 s, 156 MB/s 52428800 bytes (52 MB) copied, 0.288803 s, 182 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.256347 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355531 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.421972 s, 124 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.35316 s, 148 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.288765 s, 182 MB/s 52428800 bytes (52 MB) copied, 0.301589 s, 174 MB/s 52428800 bytes (52 MB) copied, 0.408952 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.306617 s, 171 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249178 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.370346 s, 142 MB/s 52428800 bytes (52 MB) copied, 0.438561 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19792 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211371 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.261394 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198281 s, 264 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139141 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520945 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.32906 s, 159 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.394822 s, 133 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137884 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271088 s, 193 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21491 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238312 s, 220 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642023 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138939 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157633 s, 333 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172162 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161606 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.223494 s, 235 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242423 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231076 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233826 s, 224 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153375 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149132 s, 352 MB/s 52428800 bytes (52 MB) copied, 0.121485 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1271 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12315 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064358 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831286 s, 631 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166335 s, 315 MB/s 52428800 bytes (52 MB) copied, 0.190147 s, 276 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.149154 s, 352 MB/s 52428800 bytes (52 MB) copied, 0.175696 s, 298 MB/s 52428800 bytes (52 MB) copied, 0.260722 s, 201 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162409 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283753 s, 185 MB/s 52428800 bytes (52 MB) copied, 0.150238 s, 349 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214927 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271901 s, 193 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277451 s, 189 MB/s 52428800 bytes (52 MB) copied, 0.271254 s, 193 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.268385 s, 195 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.279621 s, 187 MB/s , 0.29178 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.303314 s, 173 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242627 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240591 s, 218 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.235914 s, 222 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335993 s, 156 MB/s 52428800 bytes (52 MB) copied, 0.289519 s, 181 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.332549 s, 158 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249634 s, 210 MB/s 52428800 bytes (52 MB) copied, 0.263981 s, 199 MB/s , 0.305605 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.382262 s, 137 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.404738 s, 130 MB/s 52428800 bytes (52 MB) copied, 0.287484 s, 182 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.334532 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.345732 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.444118 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.521696 s, 100 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.471173 s, 111 MB/s 52428800 bytes (52 MB) copied, 0.48562 s, 108 MB/s 52428800 bytes (52 MB) copied, 0.34229 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191732 s, 273 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409641 s, 128 MB/s 52428800 bytes (52 MB) copied, 0.306658 s, 171 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150922 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228314 s, 230 MB/s 52428800 bytes (52 MB) copied, 0.281786 s, 186 MB/s 52428800 bytes (52 MB) copied, 0.336493 s, 156 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.289484 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367296 s, 143 MB/s 52428800 bytes (52 MB) copied, 0.139739 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158029 s, 332 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983141 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092826 s, 565 MB/s 52428800 bytes (52 MB) copied, 0.240964 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244448 s, 214 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115981 s, 452 MB/s 52428800 bytes (52 MB) copied, 0.344792 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298157 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098364 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130289 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142493 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143059 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151883 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.166361 s, 315 MB/s 52428800 bytes (52 MB) copied, 0.134287 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160452 s, 327 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160992 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161616 s, 324 MB/s 52428800 bytes (52 MB) copied, 0.208868 s, 251 MB/s , 0.148354 s, 353 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19548 s, 268 MB/s 52428800 bytes (52 MB) copied, 0.136962 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129867 s, 404 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181104 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168211 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266947 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280851 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280049 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209303 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200532 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265125 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976363 s, 537 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148091 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240466 s, 218 MB/s 52428800 bytes (52 MB) copied, 0.178666 s, 293 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260628 s, 201 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112687 s, 465 MB/s 52428800 bytes (52 MB) copied, 0.132962 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208703 s, 251 MB/s 52428800 bytes (52 MB) copied, 0.144186 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144764 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115602 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111815 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139335 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231347 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102082 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260422 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206261 s, 254 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190617 s, 275 MB/s 52428800 bytes (52 MB) copied, 0.16998 s, 308 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200073 s, 262 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198565 s, 264 MB/s 52428800 bytes (52 MB) copied, 0.193918 s, 270 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.243952 s, 215 MB/s , 0.188642 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219057 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160134 s, 327 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203094 s, 258 MB/s 52428800 bytes (52 MB) copied, 0.203355 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21109 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135336 s, 387 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161883 s, 324 MB/s 52428800 bytes (52 MB) copied, 0.199028 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.189633 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848439 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585657 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527987 s, 993 MB/s 52428800 bytes (52 MB) copied, 0.104469 s, 502 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356588 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143597 s, 365 MB/s 52428800 bytes (52 MB) copied, 0.140111 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124797 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523559 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100432 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444941 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279835 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116039 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506045 s, 1.0 GB/s , 0.02153 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584905 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664447 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756541 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285185 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356905 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273178 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440488 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295411 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659369 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683256 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129902 s, 404 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191235 s, 274 MB/s 52428800 bytes (52 MB) copied, 0.166377 s, 315 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195147 s, 269 MB/s 52428800 bytes (52 MB) copied, 0.243377 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180973 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190482 s, 275 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171656 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259104 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246559 s, 213 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.2101 s, 250 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.188888 s, 278 MB/s 50+0 records in 50+0 records out , 0.196311 s, 267 MB/s 52428800 bytes (52 MB) copied, 0.230809 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216602 s, 242 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201789 s, 260 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25158 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200904 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188685 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12745 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195774 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164431 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.268655 s, 195 MB/s 50+0 records in 50+0 records out , 0.265032 s, 198 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.308677 s, 170 MB/s 52428800 bytes (52 MB) copied, 0.239085 s, 219 MB/s 52428800 bytes (52 MB) copied, 0.264078 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182458 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746352 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164939 s, 318 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140578 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14723 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137733 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376295 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123039 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13595 s, 386 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343083 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611131 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323422 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288655 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032904 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052798 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412393 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499555 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037779 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429732 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594529 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610598 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108514 s, 483 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0928298 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134945 s, 389 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143989 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144234 s, 363 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165621 s, 317 MB/s 52428800 bytes (52 MB) copied, 0.205598 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218584 s, 240 MB/s 52428800 bytes (52 MB) copied, 0.225478 s, 233 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248814 s, 211 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279761 s, 187 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291424 s, 180 MB/s 52428800 bytes (52 MB) copied, 0.221974 s, 236 MB/s 52428800 bytes (52 MB) copied, 0.284766 s, 184 MB/s 52428800 bytes (52 MB) copied, 0.213939 s, 245 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238418 s, 220 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.34759 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258187 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.362718 s, 145 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.368895 s, 142 MB/s 52428800 bytes (52 MB) copied, 0.321082 s, 163 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199691 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.251539 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167626 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25616 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.286993 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210133 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229314 s, 229 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369212 s, 142 MB/s 52428800 bytes (52 MB) copied, 0.21253 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260632 s, 201 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953914 s, 550 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15559 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113104 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105677 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443582 s, 118 MB/s 52428800 bytes (52 MB) copied, 0.0908006 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156749 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797116 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170725 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177165 s, 296 MB/s 52428800 bytes (52 MB) copied, 0.333362 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081677 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142314 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10828 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794927 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175217 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146844 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187823 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.26376 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118211 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114875 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183752 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169797 s, 309 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222493 s, 236 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189249 s, 277 MB/s 52428800 bytes (52 MB) copied, 0.227223 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153068 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298145 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162901 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217303 s, 241 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107265 s, 489 MB/s 52428800 bytes (52 MB) copied, 0.168151 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687092 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2382 s, 220 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170688 s, 307 MB/s 52428800 bytes (52 MB) copied, 0.171821 s, 305 MB/s 52428800 bytes (52 MB) copied, 0.262751 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133123 s, 394 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879376 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187886 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725227 s, 723 MB/s 52428800 bytes (52 MB) copied, 0.0817985 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098325 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473713 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761618 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641585 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569507 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282459 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506827 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569738 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429112 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0679826 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362518 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898824 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169163 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290939 s, 180 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191226 s, 274 MB/s 52428800 bytes (52 MB) copied, 0.279751 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.310004 s, 169 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246941 s, 212 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242535 s, 216 MB/s 52428800 bytes (52 MB) copied, 0.311104 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.364461 s, 144 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25147 s, 208 MB/s 52428800 bytes (52 MB) copied, 0.177649 s, 295 MB/s 52428800 bytes (52 MB) copied, 0.155926 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17813 s, 294 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280913 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.336858 s, 156 MB/s 52428800 bytes (52 MB) copied, 0.32954 s, 159 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282672 s, 185 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.396806 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198238 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143879 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229551 s, 228 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326348 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25745 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.273718 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339291 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0990537 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187501 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228217 s, 230 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170877 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179431 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18079 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410825 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222256 s, 236 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161517 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.583061 s, 89.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201729 s, 260 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199593 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270993 s, 193 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.306072 s, 171 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.281907 s, 186 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269461 s, 195 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219715 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168365 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145452 s, 360 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201029 s, 261 MB/s 52428800 bytes (52 MB) copied, 0.207563 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161991 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339128 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245298 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225755 s, 232 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342678 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318824 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.471866 s, 111 MB/s 52428800 bytes (52 MB) copied, 0.289937 s, 181 MB/s 52428800 bytes (52 MB) copied, 0.355827 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265449 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262616 s, 200 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22793 s, 230 MB/s 52428800 bytes (52 MB) copied, 0.334116 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23777 s, 221 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290344 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.273594 s, 192 MB/s 52428800 bytes (52 MB) copied, 0.296551 s, 177 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25226 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283498 s, 185 MB/s 52428800 bytes (52 MB) copied, 0.172537 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270269 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205264 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278446 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250402 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.30231 s, 173 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.289013 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.272453 s, 192 MB/s 52428800 bytes (52 MB) copied, 0.264192 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222096 s, 236 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.236458 s, 222 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107459 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167146 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980787 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282907 s, 185 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109981 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243637 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157353 s, 333 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227447 s, 231 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230556 s, 227 MB/s 52428800 bytes (52 MB) copied, 0.243896 s, 215 MB/s 52428800 bytes (52 MB) copied, 0.209241 s, 251 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233478 s, 225 MB/s 52428800 bytes (52 MB) copied, 0.146509 s, 358 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243931 s, 215 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.313967 s, 167 MB/s 52428800 bytes (52 MB) copied, 0.146986 s, 357 MB/s 52428800 bytes (52 MB) copied, 0.141415 s, 371 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233738 s, 224 MB/s 52428800 bytes (52 MB) copied, 0.110507 s, 474 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244266 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.322508 s, 163 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.377181 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.374711 s, 140 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253822 s, 207 MB/s 52428800 bytes (52 MB) copied, 0.065655 s, 799 MB/s 52428800 bytes (52 MB) copied, 0.195495 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279479 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.316165 s, 166 MB/s 52428800 bytes (52 MB) copied, 0.270566 s, 194 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983997 s, 533 MB/s 52428800 bytes (52 MB) copied, 0.361608 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263516 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214474 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.235834 s, 222 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176798 s, 297 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.256452 s, 204 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22673 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183699 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105411 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0990821 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161845 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107559 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718659 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542098 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355694 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.257688 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195832 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269182 s, 195 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144533 s, 363 MB/s 52428800 bytes (52 MB) copied, 0.071051 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760223 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538381 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12797 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622838 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509252 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456441 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201486 s, 260 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774155 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101875 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947627 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125791 s, 417 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173433 s, 302 MB/s 52428800 bytes (52 MB) copied, 0.190224 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125877 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262048 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267416 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209046 s, 251 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242035 s, 217 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200783 s, 261 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.340092 s, 154 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out , 0.343603 s, 153 MB/s 52428800 bytes (52 MB) copied, 0.310201 s, 169 MB/s 52428800 bytes (52 MB) copied, 0.243178 s, 216 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335552 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207064 s, 253 MB/s 52428800 bytes (52 MB) copied, 0.245546 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149024 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.343482 s, 153 MB/s 52428800 bytes (52 MB) copied, 0.239128 s, 219 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.328995 s, 159 MB/s 52428800 bytes (52 MB) copied, 0.155807 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253842 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.285186 s, 184 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231503 s, 226 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.305514 s, 172 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.114791 s, 457 MB/s 52428800 bytes (52 MB) copied, 0.263731 s, 199 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.333781 s, 157 MB/s 52428800 bytes (52 MB) copied, 0.274652 s, 191 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351819 s, 149 MB/s 52428800 bytes (52 MB) copied, 0.229146 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12097 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717387 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0715702 s, 733 MB/s 52428800 bytes (52 MB) copied, 0.0883429 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641885 s, 817 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194415 s, 270 MB/s 52428800 bytes (52 MB) copied, 0.308835 s, 170 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135105 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112441 s, 466 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125248 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368336 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748207 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279049 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863353 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125058 s, 419 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119773 s, 438 MB/s 52428800 bytes (52 MB) copied, 0.158368 s, 331 MB/s 52428800 bytes (52 MB) copied, 0.129581 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1863 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196333 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212106 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222722 s, 235 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.268868 s, 195 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240657 s, 218 MB/s 52428800 bytes (52 MB) copied, 0.173881 s, 302 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359484 s, 146 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229713 s, 228 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200792 s, 261 MB/s 52428800 bytes (52 MB) copied, 0.172034 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196463 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194601 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280956 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178146 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2938 s, 178 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147274 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.31716 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147896 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244739 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693983 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107994 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122317 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184866 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933062 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355219 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100565 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0792115 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113674 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372857 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12802 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136479 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781669 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165137 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776915 s, 675 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.241743 s, 217 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126233 s, 415 MB/s 52428800 bytes (52 MB) copied, 0.216137 s, 243 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2351 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187861 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23652 s, 222 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146255 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168248 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197665 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188104 s, 279 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244153 s, 215 MB/s 52428800 bytes (52 MB) copied, 0.13774 s, 381 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215145 s, 244 MB/s 52428800 bytes (52 MB) copied, 0.127518 s, 411 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217403 s, 241 MB/s 52428800 bytes (52 MB) copied, 0.12733 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212216 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144811 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122784 s, 427 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231047 s, 227 MB/s 52428800 bytes (52 MB) copied, 0.24446 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175868 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262362 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202959 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118992 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112845 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549974 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240347 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542305 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180479 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168389 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159734 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158997 s, 330 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765129 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173257 s, 303 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044125 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655166 s, 800 MB/s 52428800 bytes (52 MB) copied, 0.0996755 s, 526 MB/s 52428800 bytes (52 MB) copied, 0.141276 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620332 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583991 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534449 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584672 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105887 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0878545 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051035 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428612 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038144 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424695 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277912 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115994 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613993 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137004 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0986687 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078161 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117548 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133295 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835107 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114899 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162147 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155115 s, 338 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105128 s, 499 MB/s 52428800 bytes (52 MB) copied, 0.109949 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161673 s, 324 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187397 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182866 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180787 s, 290 MB/s 52428800 bytes (52 MB) copied, 0.265814 s, 197 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.221184 s, 237 MB/s 52428800 bytes (52 MB) copied, 0.160134 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275953 s, 190 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242284 s, 216 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185383 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181806 s, 288 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234942 s, 223 MB/s , 0.317403 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143361 s, 366 MB/s 52428800 bytes (52 MB) copied, 0.247564 s, 212 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.148755 s, 352 MB/s , 0.289976 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232829 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208015 s, 252 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122117 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125155 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259527 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148505 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667344 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717275 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417747 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258856 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321509 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357042 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0427362 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0861804 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628553 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793735 s, 661 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12725 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843125 s, 622 MB/s 52428800 bytes (52 MB) copied, 0.127388 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060219 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143773 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964313 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122816 s, 427 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932347 s, 562 MB/s 52428800 bytes (52 MB) copied, 0.13171 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212986 s, 246 MB/s , 0.204897 s, 256 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191044 s, 274 MB/s 52428800 bytes (52 MB) copied, 0.161343 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106085 s, 494 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194868 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204856 s, 256 MB/s 52428800 bytes (52 MB) copied, 0.235784 s, 222 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.235021 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187742 s, 279 MB/s 52428800 bytes (52 MB) copied, 0.230079 s, 228 MB/s 52428800 bytes (52 MB) copied, 0.211734 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234956 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224831 s, 233 MB/s , 0.291434 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186385 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209948 s, 250 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606052 s, 865 MB/s 52428800 bytes (52 MB) copied, 0.14095 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033916 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801565 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201016 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336581 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237368 s, 221 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103513 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155796 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218606 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590476 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030731 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055679 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835454 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581677 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659749 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0945257 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345336 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131464 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167353 s, 313 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218299 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213226 s, 246 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.223929 s, 234 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175851 s, 298 MB/s 52428800 bytes (52 MB) copied, 0.151721 s, 346 MB/s 52428800 bytes (52 MB) copied, 0.239851 s, 219 MB/s 52428800 bytes (52 MB) copied, 0.221613 s, 237 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.221507 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.257164 s, 204 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25576 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209215 s, 251 MB/s 52428800 bytes (52 MB) copied, 0.209271 s, 251 MB/s 52428800 bytes (52 MB) copied, 0.310162 s, 169 MB/s 52428800 bytes (52 MB) copied, 0.203119 s, 258 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.256561 s, 204 MB/s 52428800 bytes (52 MB) copied, 0.257809 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211046 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219337 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.376553 s, 139 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209688 s, 250 MB/s 52428800 bytes (52 MB) copied, 0.202232 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170271 s, 308 MB/s 52428800 bytes (52 MB) copied, 0.250835 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175697 s, 298 MB/s 52428800 bytes (52 MB) copied, 0.204175 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230502 s, 227 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162201 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824149 s, 636 MB/s 52428800 bytes (52 MB) copied, 0.158478 s, 331 MB/s 52428800 bytes (52 MB) copied, 0.319953 s, 164 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161332 s, 325 MB/s 52428800 bytes (52 MB) copied, 0.151931 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120764 s, 434 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107978 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217415 s, 241 MB/s 52428800 bytes (52 MB) copied, 0.182153 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231926 s, 226 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184981 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191491 s, 274 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207566 s, 253 MB/s 52428800 bytes (52 MB) copied, 0.203609 s, 257 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226519 s, 231 MB/s 52428800 bytes (52 MB) copied, 0.190295 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11971 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238645 s, 220 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269182 s, 195 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.325236 s, 161 MB/s 52428800 bytes (52 MB) copied, 0.21776 s, 241 MB/s 52428800 bytes (52 MB) copied, 0.215471 s, 243 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15622 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174544 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835023 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367994 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141079 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.273927 s, 191 MB/s , 0.191338 s, 274 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110536 s, 474 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237435 s, 221 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267209 s, 196 MB/s 52428800 bytes (52 MB) copied, 0.41756 s, 126 MB/s 52428800 bytes (52 MB) copied, 0.288995 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143244 s, 366 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20281 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182038 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225295 s, 233 MB/s 52428800 bytes (52 MB) copied, 0.605778 s, 86.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.289298 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246399 s, 213 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188123 s, 279 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259622 s, 202 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129514 s, 405 MB/s 52428800 bytes (52 MB) copied, 0.381377 s, 137 MB/s 52428800 bytes (52 MB) copied, 0.299511 s, 175 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.300903 s, 174 MB/s 52428800 bytes (52 MB) copied, 0.233353 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.36372 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22383 s, 234 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.510738 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.388914 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.383509 s, 137 MB/s , 0.477316 s, 110 MB/s 52428800 bytes (52 MB) copied, 0.319454 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265825 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193434 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186587 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213996 s, 245 MB/s , 0.287044 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233218 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326438 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130635 s, 401 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222123 s, 236 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23559 s, 223 MB/s 52428800 bytes (52 MB) copied, 0.185172 s, 283 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179859 s, 291 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112342 s, 467 MB/s 52428800 bytes (52 MB) copied, 0.219424 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164413 s, 319 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248313 s, 211 MB/s 52428800 bytes (52 MB) copied, 0.191115 s, 274 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14676 s, 357 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143809 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174168 s, 301 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167743 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056735 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162839 s, 322 MB/s , 0.114168 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201169 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138658 s, 378 MB/s , 0.238534 s, 220 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18684 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994328 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873112 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187236 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215244 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138711 s, 378 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161418 s, 325 MB/s 52428800 bytes (52 MB) copied, 0.257821 s, 203 MB/s 52428800 bytes (52 MB) copied, 0.274248 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156237 s, 336 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216078 s, 243 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180623 s, 290 MB/s 52428800 bytes (52 MB) copied, 0.168454 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924648 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174194 s, 301 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582149 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170514 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.19933 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.354593 s, 148 MB/s 52428800 bytes (52 MB) copied, 0.181952 s, 288 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.27444 s, 191 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.182239 s, 288 MB/s , 0.168716 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175724 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194145 s, 270 MB/s 52428800 bytes (52 MB) copied, 0.182328 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202391 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224181 s, 234 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150932 s, 347 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10434 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120038 s, 437 MB/s 52428800 bytes (52 MB) copied, 0.22725 s, 231 MB/s 52428800 bytes (52 MB) copied, 0.139497 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157654 s, 333 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730897 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971045 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480029 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106041 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12511 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194096 s, 270 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240231 s, 218 MB/s 52428800 bytes (52 MB) copied, 0.17908 s, 293 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.230986 s, 227 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200799 s, 261 MB/s 52428800 bytes (52 MB) copied, 0.245702 s, 213 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264147 s, 198 MB/s , 0.239272 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298219 s, 176 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282092 s, 186 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.282602 s, 186 MB/s 52428800 bytes (52 MB) copied, 0.279569 s, 188 MB/s 52428800 bytes (52 MB) copied, 0.289287 s, 181 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.367873 s, 143 MB/s , 0.3128 s, 168 MB/s 52428800 bytes (52 MB) copied, 0.259445 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.323412 s, 162 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.37661 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.356931 s, 147 MB/s 52428800 bytes (52 MB) copied, 0.307112 s, 171 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348096 s, 151 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.376588 s, 139 MB/s 52428800 bytes (52 MB) copied, 0.244926 s, 214 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.405971 s, 129 MB/s , 0.339918 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.261867 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233181 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277939 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.288815 s, 182 MB/s 52428800 bytes (52 MB) copied, 0.309545 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263969 s, 199 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.227419 s, 231 MB/s , 0.234917 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16873 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0878981 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0852546 s, 615 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134245 s, 391 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809506 s, 648 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824529 s, 636 MB/s 52428800 bytes (52 MB) copied, 0.115923 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131873 s, 398 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159745 s, 328 MB/s 52428800 bytes (52 MB) copied, 0.196937 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120321 s, 436 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177928 s, 295 MB/s 52428800 bytes (52 MB) copied, 0.187931 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181503 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108187 s, 485 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189147 s, 277 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.220717 s, 238 MB/s 52428800 bytes (52 MB) copied, 0.324772 s, 161 MB/s 52428800 bytes (52 MB) copied, 0.219255 s, 239 MB/s 52428800 bytes (52 MB) copied, 0.291473 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262584 s, 200 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206396 s, 254 MB/s 52428800 bytes (52 MB) copied, 0.251245 s, 209 MB/s 52428800 bytes (52 MB) copied, 0.271401 s, 193 MB/s 52428800 bytes (52 MB) copied, 0.2276 s, 230 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228179 s, 230 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054913 s, 955 MB/s 52428800 bytes (52 MB) copied, 0.219425 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871092 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180663 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.276778 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177089 s, 296 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186754 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12888 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112517 s, 466 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194985 s, 269 MB/s 52428800 bytes (52 MB) copied, 0.121882 s, 430 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132463 s, 396 MB/s 52428800 bytes (52 MB) copied, 0.127505 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202247 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147077 s, 356 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149826 s, 350 MB/s 52428800 bytes (52 MB) copied, 0.0906532 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120287 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770247 s, 681 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121844 s, 430 MB/s 52428800 bytes (52 MB) copied, 0.220201 s, 238 MB/s 52428800 bytes (52 MB) copied, 0.21486 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209105 s, 251 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.32081 s, 163 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23741 s, 221 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195211 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188309 s, 278 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177618 s, 295 MB/s 52428800 bytes (52 MB) copied, 0.23324 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243271 s, 216 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1994 s, 263 MB/s 52428800 bytes (52 MB) copied, 0.184906 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240438 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153677 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224743 s, 233 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244587 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229905 s, 228 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207078 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214111 s, 245 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2225 s, 236 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209711 s, 250 MB/s 52428800 bytes (52 MB) copied, 0.1659 s, 316 MB/s 52428800 bytes (52 MB) copied, 0.210218 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.254405 s, 206 MB/s 52428800 bytes (52 MB) copied, 0.227554 s, 230 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196914 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065007 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280299 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153172 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176629 s, 297 MB/s 52428800 bytes (52 MB) copied, 0.19567 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144065 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164089 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121208 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789216 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160339 s, 327 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105219 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994236 s, 527 MB/s 52428800 bytes (52 MB) copied, 0.154116 s, 340 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127334 s, 412 MB/s 52428800 bytes (52 MB) copied, 0.0894186 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200854 s, 261 MB/s 52428800 bytes (52 MB) copied, 0.161177 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214254 s, 245 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260542 s, 201 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.312722 s, 168 MB/s 52428800 bytes (52 MB) copied, 0.330588 s, 159 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.220894 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.254677 s, 206 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.381418 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.281473 s, 186 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271999 s, 193 MB/s 52428800 bytes (52 MB) copied, 0.262689 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109485 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.332061 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.27793 s, 189 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269715 s, 194 MB/s 52428800 bytes (52 MB) copied, 0.373409 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.308597 s, 170 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.220815 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260845 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.375639 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206447 s, 254 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.274439 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33389 s, 157 MB/s 52428800 bytes (52 MB) copied, 0.539413 s, 97.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127442 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154074 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.434136 s, 121 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275004 s, 191 MB/s 52428800 bytes (52 MB) copied, 0.236624 s, 222 MB/s 52428800 bytes (52 MB) copied, 0.145085 s, 361 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164998 s, 318 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112768 s, 465 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109078 s, 481 MB/s , 0.315037 s, 166 MB/s 52428800 bytes (52 MB) copied, 0.12087 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131974 s, 397 MB/s 52428800 bytes (52 MB) copied, 0.18626 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132635 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662675 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14541 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242006 s, 217 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653011 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399724 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706412 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279286 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106559 s, 492 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104506 s, 502 MB/s 52428800 bytes (52 MB) copied, 0.104163 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14721 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186613 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205081 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13923 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13281 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15393 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158544 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196955 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162308 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181106 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191863 s, 273 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182559 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237583 s, 221 MB/s 52428800 bytes (52 MB) copied, 0.155738 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212437 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205382 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.220779 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197729 s, 265 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.204911 s, 256 MB/s , 0.2972 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129921 s, 404 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141134 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199569 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127586 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243246 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134577 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124797 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174086 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25106 s, 209 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202972 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191272 s, 274 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20037 s, 262 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228775 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361142 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907143 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304243 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470603 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383989 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304299 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579213 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035832 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533194 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494416 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455437 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0916551 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489167 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941206 s, 557 MB/s 52428800 bytes (52 MB) copied, 0.0948193 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127041 s, 413 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15353 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131613 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174689 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17798 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24263 s, 216 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20334 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.25313 s, 207 MB/s 52428800 bytes (52 MB) copied, 0.31098 s, 169 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.315257 s, 166 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200964 s, 261 MB/s 52428800 bytes (52 MB) copied, 0.25955 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249797 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.325951 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195847 s, 268 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348661 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298885 s, 175 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.294695 s, 178 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.400485 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258896 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231408 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456151 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.422684 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.379583 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228952 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.372982 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.396479 s, 132 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237274 s, 221 MB/s , 0.242765 s, 216 MB/s 52428800 bytes (52 MB) copied, 0.139883 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292772 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969445 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550388 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.316179 s, 166 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224933 s, 233 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120178 s, 436 MB/s 52428800 bytes (52 MB) copied, 0.0643527 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160671 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0713904 s, 734 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155365 s, 337 MB/s 52428800 bytes (52 MB) copied, 0.0760031 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100525 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992173 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114305 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883407 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691453 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18388 s, 285 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147842 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818206 s, 641 MB/s 52428800 bytes (52 MB) copied, 0.131768 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21444 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107398 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153522 s, 342 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197025 s, 266 MB/s 52428800 bytes (52 MB) copied, 0.198245 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207191 s, 253 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.208783 s, 251 MB/s 52428800 bytes (52 MB) copied, 0.179725 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.20579 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2026 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205033 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201479 s, 260 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16724 s, 313 MB/s 52428800 bytes (52 MB) copied, 0.15968 s, 328 MB/s 52428800 bytes (52 MB) copied, 0.354271 s, 148 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.307118 s, 171 MB/s 52428800 bytes (52 MB) copied, 0.169416 s, 309 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.272912 s, 192 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145231 s, 361 MB/s 52428800 bytes (52 MB) copied, 0.244951 s, 214 MB/s 52428800 bytes (52 MB) copied, 0.184783 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.3321 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.293901 s, 178 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219426 s, 239 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21399 s, 245 MB/s 52428800 bytes (52 MB) copied, 0.222522 s, 236 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203364 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218225 s, 240 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152062 s, 345 MB/s 52428800 bytes (52 MB) copied, 0.277207 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152641 s, 343 MB/s 52428800 bytes (52 MB) copied, 0.161315 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122135 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197298 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106825 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136996 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130885 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455367 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465133 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027223 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443675 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048182 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360515 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625005 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488688 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696774 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074918 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406766 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082301 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847781 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820742 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111748 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111814 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108971 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126676 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.212207 s, 247 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136741 s, 383 MB/s 52428800 bytes (52 MB) copied, 0.218985 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164098 s, 319 MB/s , 0.161046 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137849 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141601 s, 370 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177168 s, 296 MB/s 52428800 bytes (52 MB) copied, 0.143382 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187356 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210706 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187563 s, 280 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176362 s, 297 MB/s 52428800 bytes (52 MB) copied, 0.204317 s, 257 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127513 s, 411 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169601 s, 309 MB/s 52428800 bytes (52 MB) copied, 0.177028 s, 296 MB/s 52428800 bytes (52 MB) copied, 0.132668 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201518 s, 260 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188951 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186737 s, 281 MB/s 52428800 bytes (52 MB) copied, 0.11437 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853636 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711851 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144483 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196733 s, 266 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16299 s, 322 MB/s 52428800 bytes (52 MB) copied, 0.125261 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173841 s, 302 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05071 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122069 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816673 s, 642 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104321 s, 503 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115922 s, 452 MB/s , 0.0875062 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539537 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901662 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292204 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090235 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430808 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642586 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548907 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407267 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345636 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378899 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690056 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122599 s, 428 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0892322 s, 588 MB/s , 0.0785666 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118458 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127636 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144582 s, 363 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149707 s, 350 MB/s 52428800 bytes (52 MB) copied, 0.139048 s, 377 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949605 s, 552 MB/s 52428800 bytes (52 MB) copied, 0.108842 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118156 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133442 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198702 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144103 s, 364 MB/s 50+0 records in 50+0 records out , 0.159602 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983394 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229192 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21254 s, 247 MB/s 52428800 bytes (52 MB) copied, 0.197234 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09491 s, 552 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141077 s, 372 MB/s 52428800 bytes (52 MB) copied, 0.228489 s, 229 MB/s , 0.277964 s, 189 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788056 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204245 s, 257 MB/s 52428800 bytes (52 MB) copied, 0.076012 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685135 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178908 s, 293 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207595 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198042 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134476 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134455 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543956 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.097665 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780404 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746537 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0993867 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450245 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618041 s, 848 MB/s 52428800 bytes (52 MB) copied, 0.070225 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111291 s, 471 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123389 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143586 s, 365 MB/s 52428800 bytes (52 MB) copied, 0.126585 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119365 s, 439 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130259 s, 402 MB/s 52428800 bytes (52 MB) copied, 0.116769 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160922 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137244 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144391 s, 363 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167358 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163091 s, 321 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103214 s, 508 MB/s , 0.141779 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139342 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162867 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120281 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0834629 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167564 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120288 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14188 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173936 s, 301 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153086 s, 342 MB/s 52428800 bytes (52 MB) copied, 0.112831 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702668 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0987846 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901075 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108348 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864112 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139482 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643209 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100471 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910996 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931798 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137648 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644438 s, 814 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0977081 s, 537 MB/s 52428800 bytes (52 MB) copied, 0.0845521 s, 620 MB/s 52428800 bytes (52 MB) copied, 0.0958207 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142798 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190951 s, 275 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153442 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160947 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117098 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198737 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198094 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121196 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193821 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159317 s, 329 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16676 s, 314 MB/s 52428800 bytes (52 MB) copied, 0.145663 s, 360 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111043 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168692 s, 311 MB/s 52428800 bytes (52 MB) copied, 0.153205 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232408 s, 226 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141093 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141779 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138335 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11781 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946458 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049018 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076905 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180901 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105594 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472643 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327625 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261882 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0902506 s, 581 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969306 s, 541 MB/s 52428800 bytes (52 MB) copied, 0.0727417 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110232 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226136 s, 232 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184235 s, 285 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.220906 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176364 s, 297 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227277 s, 231 MB/s 52428800 bytes (52 MB) copied, 0.189646 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215236 s, 244 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174692 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.235601 s, 223 MB/s 52428800 bytes (52 MB) copied, 0.25505 s, 206 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211675 s, 248 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250583 s, 209 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239881 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228399 s, 230 MB/s 52428800 bytes (52 MB) copied, 0.179051 s, 293 MB/s 52428800 bytes (52 MB) copied, 0.289312 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132142 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222896 s, 235 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163247 s, 321 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163286 s, 321 MB/s 52428800 bytes (52 MB) copied, 0.149043 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259621 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135869 s, 386 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176048 s, 298 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121583 s, 431 MB/s 52428800 bytes (52 MB) copied, 0.102614 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145737 s, 360 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0902106 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135166 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481847 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217788 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539838 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647562 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196302 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435814 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125762 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328548 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481072 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511089 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336069 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056837 s, 922 MB/s 52428800 bytes (52 MB) copied, 0.0684521 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718734 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591516 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439208 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423155 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653106 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103227 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360785 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520436 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753459 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483092 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519584 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503509 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389783 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462061 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437338 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539721 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923067 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111794 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112686 s, 465 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12139 s, 432 MB/s 52428800 bytes (52 MB) copied, 0.149031 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105948 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943165 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124791 s, 420 MB/s , 0.107049 s, 490 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922428 s, 568 MB/s 52428800 bytes (52 MB) copied, 0.117049 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151927 s, 345 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904689 s, 580 MB/s 52428800 bytes (52 MB) copied, 0.147922 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847135 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079069 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145443 s, 360 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.223711 s, 234 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931424 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922528 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125014 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117975 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145159 s, 361 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755845 s, 694 MB/s 52428800 bytes (52 MB) copied, 0.0760776 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103977 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153901 s, 341 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21946 s, 239 MB/s 52428800 bytes (52 MB) copied, 0.0593674 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150073 s, 349 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494487 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0338577 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0866291 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726749 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604477 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562492 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441752 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441541 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316582 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224383 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553998 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364478 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236703 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040197 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106965 s, 490 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117097 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100253 s, 523 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132469 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116357 s, 451 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196888 s, 266 MB/s 50+0 records in 50+0 records out , 0.149934 s, 350 MB/s 52428800 bytes (52 MB) copied, 0.10487 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161029 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154079 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12259 s, 428 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189769 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.312368 s, 168 MB/s 52428800 bytes (52 MB) copied, 0.160462 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165452 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198686 s, 264 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.234918 s, 223 MB/s 52428800 bytes (52 MB) copied, 0.248531 s, 211 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240928 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.293605 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409307 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232663 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.301718 s, 174 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.222316 s, 236 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.338132 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369058 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.24262 s, 216 MB/s 52428800 bytes (52 MB) copied, 0.142012 s, 369 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.34175 s, 153 MB/s 52428800 bytes (52 MB) copied, 0.344433 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.366049 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.304743 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.53939 s, 97.2 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335162 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240546 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112165 s, 467 MB/s 52428800 bytes (52 MB) copied, 0.248436 s, 211 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133593 s, 392 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249934 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355102 s, 148 MB/s 52428800 bytes (52 MB) copied, 0.279255 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149452 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105461 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227895 s, 230 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.353031 s, 149 MB/s , 0.185763 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119323 s, 439 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164557 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113021 s, 464 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198342 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1597 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175657 s, 298 MB/s 52428800 bytes (52 MB) copied, 0.161961 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227455 s, 231 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22518 s, 233 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100835 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210218 s, 249 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0646995 s, 810 MB/s , 0.0760728 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537932 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585325 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193484 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154831 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490378 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373676 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451995 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106435 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111085 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104865 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110749 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175267 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228626 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225886 s, 232 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191597 s, 274 MB/s 52428800 bytes (52 MB) copied, 0.154386 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124961 s, 420 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205098 s, 256 MB/s 52428800 bytes (52 MB) copied, 0.180743 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.276124 s, 190 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.240138 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.191962 s, 273 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.236811 s, 221 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196931 s, 266 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.285748 s, 183 MB/s , 0.164058 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152891 s, 343 MB/s 52428800 bytes (52 MB) copied, 0.356134 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193452 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186416 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155021 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.293203 s, 179 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21662 s, 242 MB/s 52428800 bytes (52 MB) copied, 0.109952 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249709 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.325438 s, 161 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170904 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134997 s, 388 MB/s 52428800 bytes (52 MB) copied, 0.14828 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245699 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376728 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535024 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435095 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513856 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433685 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597518 s, 877 MB/s 52428800 bytes (52 MB) copied, 0.102493 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103927 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106314 s, 493 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121891 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124033 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134839 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159453 s, 329 MB/s 52428800 bytes (52 MB) copied, 0.120858 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123165 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113751 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217877 s, 241 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239437 s, 219 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171762 s, 305 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210175 s, 249 MB/s 52428800 bytes (52 MB) copied, 0.24368 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290898 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229199 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277548 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239992 s, 218 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173673 s, 302 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171882 s, 305 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.213435 s, 246 MB/s , 0.169537 s, 309 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258848 s, 203 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195935 s, 268 MB/s 52428800 bytes (52 MB) copied, 0.175359 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183045 s, 286 MB/s 52428800 bytes (52 MB) copied, 0.168057 s, 312 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633005 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1945 s, 270 MB/s 52428800 bytes (52 MB) copied, 0.0683442 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139941 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131482 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253567 s, 207 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239071 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216834 s, 242 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144953 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025321 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415775 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542708 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441836 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847827 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688373 s, 762 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847197 s, 619 MB/s 52428800 bytes (52 MB) copied, 0.155446 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.156297 s, 335 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142313 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.123436 s, 425 MB/s , 0.189076 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155692 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179609 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127779 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190944 s, 275 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262116 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226192 s, 232 MB/s 52428800 bytes (52 MB) copied, 0.172826 s, 303 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279662 s, 187 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21102 s, 248 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.171979 s, 305 MB/s 52428800 bytes (52 MB) copied, 0.213788 s, 245 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.302911 s, 173 MB/s , 0.197169 s, 266 MB/s 52428800 bytes (52 MB) copied, 0.195164 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.253566 s, 207 MB/s 52428800 bytes (52 MB) copied, 0.183326 s, 286 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.255781 s, 205 MB/s 52428800 bytes (52 MB) copied, 0.239275 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167722 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228866 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170517 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.190136 s, 276 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152684 s, 343 MB/s 52428800 bytes (52 MB) copied, 0.293926 s, 178 MB/s 52428800 bytes (52 MB) copied, 0.15548 s, 337 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142442 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966527 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115714 s, 453 MB/s 52428800 bytes (52 MB) copied, 0.0753132 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12636 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.306289 s, 171 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549645 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054927 s, 955 MB/s , 0.092087 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783948 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347483 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034355 s, 1.5 GB/s , 0.0773526 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223609 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132281 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309713 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745051 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.11874 s, 442 MB/s 52428800 bytes (52 MB) copied, 0.0314171 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022589 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764578 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490647 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561738 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0792472 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116053 s, 452 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108999 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0930199 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830799 s, 631 MB/s 52428800 bytes (52 MB) copied, 0.0900893 s, 582 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116901 s, 448 MB/s 52428800 bytes (52 MB) copied, 0.120091 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144733 s, 362 MB/s 52428800 bytes (52 MB) copied, 0.130999 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115569 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113124 s, 463 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142534 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11551 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132922 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124045 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146535 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141398 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564171 s, 929 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116838 s, 449 MB/s 52428800 bytes (52 MB) copied, 0.0697821 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104182 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520842 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571751 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668467 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351985 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369821 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036673 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467278 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043603 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366152 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456913 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470274 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036302 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664276 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870771 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0823678 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670197 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0902785 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771197 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148668 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189661 s, 276 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173787 s, 302 MB/s 52428800 bytes (52 MB) copied, 0.140741 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167434 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155985 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.220723 s, 238 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228147 s, 230 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164404 s, 319 MB/s 52428800 bytes (52 MB) copied, 0.220411 s, 238 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187828 s, 279 MB/s 52428800 bytes (52 MB) copied, 0.199572 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277843 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141588 s, 370 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265405 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131301 s, 399 MB/s 52428800 bytes (52 MB) copied, 0.116687 s, 449 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237937 s, 220 MB/s 52428800 bytes (52 MB) copied, 0.152477 s, 344 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.293995 s, 178 MB/s 52428800 bytes (52 MB) copied, 0.28217 s, 186 MB/s 52428800 bytes (52 MB) copied, 0.187006 s, 280 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609969 s, 860 MB/s 52428800 bytes (52 MB) copied, 0.209508 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149537 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132328 s, 396 MB/s , 0.129431 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153375 s, 342 MB/s 52428800 bytes (52 MB) copied, 0.105246 s, 498 MB/s , 0.303521 s, 173 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124586 s, 421 MB/s 52428800 bytes (52 MB) copied, 0.191479 s, 274 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992177 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090207 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205243 s, 255 MB/s 52428800 bytes (52 MB) copied, 0.175623 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206993 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886882 s, 591 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864249 s, 607 MB/s 52428800 bytes (52 MB) copied, 0.193572 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090861 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127634 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123523 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157133 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877337 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588014 s, 892 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835279 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475264 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475249 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.140531 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167744 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590806 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494005 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526179 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531633 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611199 s, 858 MB/s /mnt/lustre/f45.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x280000401:0x185:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800b5fc9000.stats= snapshot_time 1746159981.385435735 secs.nsecs start_time 1746159920.778280668 secs.nsecs elapsed_time 60.607155067 secs.nsecs read_bytes 2509 samples [bytes] 4096 1048576 2621440000 2745869262651392 read 97000 samples [usecs] 159 469879 319797090 11931984842638 ioctl 1 samples [reqs] open 1941 samples [usecs] 5 95862 603115 26037762007 close 1941 samples [usecs] 4 104259 753772 25250461924 seek 1939 samples [usecs] 2 965 12479 1014975 setattr 1 samples [usecs] 21852 21852 21852 477509904 getattr 32 samples [usecs] 133 13860 26486 235011396 mknod 1 samples [usecs] 8604 8604 8604 74028816 inode_permission 7739 samples [usecs] 1 49805 1590326 13929300258 opencount 1942 samples [reqs] 1 1940 1883825 2436787707 openclosetime 1940 samples [usecs] 25 284583 64672476 4366266028848 pcc_attach 1 samples [reqs] pcc_hit_bytes 1890 samples [bytes] 52428800 52428800 99090432000 5195192441241600000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff88012d5b8000.stats= snapshot_time 1746159981.386997896 secs.nsecs start_time 1746159920.778306086 secs.nsecs elapsed_time 60.608691810 secs.nsecs read_bytes 1154 samples [bytes] 4096 1048576 1205862400 1263286716727296 read 94300 samples [usecs] 159 794179 336272206 21389056605888 open 1886 samples [usecs] 4 33128 225529 3567905081 close 1886 samples [usecs] 4 90986 731988 26205092124 seek 1885 samples [usecs] 2 37247 95589 2550404649 inode_permission 3770 samples [usecs] 1 55699 849918 12700285908 opencount 1886 samples [reqs] 1 1870 1749401 2181483111 openclosetime 1869 samples [usecs] 16 263497 65017614 4598115530924 pcc_attach 1 samples [reqs] pcc_hit_bytes 1863 samples [bytes] 52428800 52428800 97674854400 5120975406366720000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 attach 2 detach 0 auto_attach 0 llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1073741824 llite.lustre-ffff88012d5b8000.pcc_async_threshold=1073741824 /mnt/pcc.d45.sanity-pcc.2 is idle now /mnt/pcc.d45.sanity-pcc is idle now PASS 45 (68s) == sanity-pcc test 46: Verify PCC mode setting works correctly ========================================================== 00:26:25 (1746159985) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0897807 s, 701 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d46.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 24f6053c-ff99-4fb7-85ba-39985734dbec Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f46.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=24f6053c-ff99-4fb7-85ba-39985734dbec (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d46.sanity-pcc/d46.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} running as uid/gid/euid/egid 500/500/500/500, groups: 0 [id] uid=500(sanityusr) gid=500(sanityusr) groups=500(sanityusr),0(root) Mode: 0400 llite.lustre-ffff8800b5fc9000.pcc_mode=0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800b5fc9000.pcc_mode=0400 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800b5fc9000.pcc_mode=0004 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [stat] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 8 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0664/-rw-rw-r--) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-02 00:26:27.000000000 -0400 Modify: 2025-05-02 00:26:27.000000000 -0400 Change: 2025-05-02 00:26:27.000000000 -0400 Birth: - running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800b5fc9000.pcc_mode=0040 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 4 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0660/-rw-rw----) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-02 00:26:28.000000000 -0400 Modify: 2025-05-02 00:26:27.000000000 -0400 Change: 2025-05-02 00:26:28.000000000 -0400 Birth: - file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800b5fc9000.pcc_mode=0400 llite.lustre-ffff8800b5fc9000.pcc_mode=0400 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d46.sanity-pcc is idle now PASS 46 (7s) == sanity-pcc test 47: mtime should be kept once file attached into PCC ========================================================== 00:26:32 (1746159992) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.188041 s, 335 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d47.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: cd1277db-ffa7-40e0-9a28-a58d2351f25a Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f47.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=cd1277db-ffa7-40e0-9a28-a58d2351f25a (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d47.sanity-pcc/d47.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} QQQQQ /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/f47.sanity-pcc | grep -E -c 'type: readonly' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d47.sanity-pcc is idle now PASS 47 (9s) == sanity-pcc test 48: PCC state should check whether the file in local PCC cache ========================================================== 00:26:41 (1746160001) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.111 s, 567 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d48.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: e1b218de-6bbf-4a72-bb62-3217baf1ed46 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f48.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e1b218de-6bbf-4a72-bb62-3217baf1ed46 (extents) (64bit) (large files) (huge files) QQQQQ file: /mnt/lustre/f48.sanity-pcc, type: readonly, PCC_file: /d48.sanity-pcc/59fd/0x2000059f1:0xc:0x0 (unlinked), open_count: 0, flags: 4 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d48.sanity-pcc is idle now PASS 48 (7s) == sanity-pcc test 96: Auto attach from multiple read process on a node ========================================================== 00:26:48 (1746160008) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.121416 s, 518 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d96.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: c8c59902-617e-4b8d-b591-3e723c0390af Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f96.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c8c59902-617e-4b8d-b591-3e723c0390af (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d96.sanity-pcc/d96.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1G llite.lustre-ffff88012d5b8000.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.751658 s, 69.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.895146 s, 58.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.913982 s, 57.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.981842 s, 53.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.982021 s, 53.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.998518 s, 52.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389468 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436064 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316169 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271458 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478564 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543126 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486452 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721172 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526755 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471538 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446007 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364794 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381293 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386896 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641953 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055019 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946879 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329997 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444089 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454278 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642228 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410716 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411141 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256741 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249811 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236136 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407851 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472642 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509137 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475521 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371945 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800773 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593628 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335358 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062675 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513487 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318781 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485108 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528109 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592953 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399371 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449852 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483607 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447576 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647782 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481313 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445968 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036497 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376376 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0887898 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441142 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550524 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508972 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419737 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421544 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521644 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229524 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549752 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332746 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479985 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316452 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614975 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439058 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279594 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308457 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757178 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737272 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0990647 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558429 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310357 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340667 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467319 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075049 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335255 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443922 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349808 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324604 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705582 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571985 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531592 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609592 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367369 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482887 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573621 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825008 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555561 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697071 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372878 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831141 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328141 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312036 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409964 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247161 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451533 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334525 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476946 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500615 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486208 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385464 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271464 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550174 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462583 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450961 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299308 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334555 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046176 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677208 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047326 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488066 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468607 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563466 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04598 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452356 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376507 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03271 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835647 s, 627 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118118 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853661 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281335 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358533 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262299 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294856 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447637 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748353 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849452 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0636037 s, 824 MB/s 52428800 bytes (52 MB) copied, 0.048519 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020884 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059011 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452062 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513964 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0955962 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542651 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573082 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293494 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563991 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589088 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379628 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458426 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592334 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525438 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327561 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480165 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310359 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498917 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395348 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327077 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442379 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270566 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0590599 s, 888 MB/s , 0.0365708 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432798 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446848 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053228 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312368 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214659 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850912 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403991 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296961 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336229 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111182 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537858 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391989 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411631 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475003 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077985 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319205 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057434 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297338 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565958 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537688 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575047 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395896 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402217 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327367 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054949 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468994 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428673 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404746 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344857 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258544 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357376 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039166 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731088 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929108 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377849 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402503 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318056 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549662 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492482 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038656 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369901 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425804 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372633 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426916 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419386 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420912 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551344 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497489 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320885 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512434 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618705 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392378 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493003 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548728 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309558 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337783 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796039 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967101 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603473 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560894 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341221 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066687 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053551 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268999 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369254 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355958 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368781 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294502 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331835 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095003 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477471 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051329 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525971 s, 997 MB/s , 0.041989 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541388 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218258 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509773 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048541 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437508 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060505 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381447 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454916 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484876 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040978 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361567 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203663 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266303 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353449 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0640243 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038564 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496758 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361925 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433828 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495761 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539364 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492155 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625014 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542798 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465175 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060754 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447007 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442446 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875677 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398324 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425404 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436177 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703674 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608117 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045881 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367664 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716663 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358348 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565954 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276422 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770338 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376921 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722746 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734394 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389831 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036012 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641702 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684874 s, 766 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603868 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347144 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475422 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545225 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555519 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731562 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628066 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530774 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056099 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258291 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518486 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349024 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388608 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613122 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347325 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588121 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277336 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094786 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034063 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370293 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464167 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364506 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049856 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493185 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431021 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405873 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457071 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360291 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368982 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372687 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389149 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450779 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427549 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273987 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045058 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215404 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252623 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418913 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500423 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425057 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036057 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368136 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055359 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553459 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485708 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495845 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453524 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388684 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441438 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524006 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430734 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042442 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308719 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311318 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378515 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407505 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729035 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331502 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608998 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680704 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810277 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427165 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613094 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470045 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579719 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686286 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435446 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519063 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441947 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311036 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357739 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597758 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437037 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522134 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490294 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497886 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053717 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291398 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047385 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544699 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352311 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364838 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611704 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422564 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574092 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405004 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487703 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372444 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444025 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365397 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187379 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312721 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484022 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820896 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231058 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285843 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242401 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256571 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230937 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433321 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402731 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403832 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522558 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584639 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036375 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269298 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481608 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418702 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550167 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453626 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265306 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364088 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022767 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364425 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247726 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275016 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530447 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221864 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441992 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277839 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413189 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380584 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370904 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616584 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027635 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448519 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400117 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690401 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421681 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703381 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093272 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028182 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349308 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431326 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247086 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312903 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316981 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190391 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466536 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0956234 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868594 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233933 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508273 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040922 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314228 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355005 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493882 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498362 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602989 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707643 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360738 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048815 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306021 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216657 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444327 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289052 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359543 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343115 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030007 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382488 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219549 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258896 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373747 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347532 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318729 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383904 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285453 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023434 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383765 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556174 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567326 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194718 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357068 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254379 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039726 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324069 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361214 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566068 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329691 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305356 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505262 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035167 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492337 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321194 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508132 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056004 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296685 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336356 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380407 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334542 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439684 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431897 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057391 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302397 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535763 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389485 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436565 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532732 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320114 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391971 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212838 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314812 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366995 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254184 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029681 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030333 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277455 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180564 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294086 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161885 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225034 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189475 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368227 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061974 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511595 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440488 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317951 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396189 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475992 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303821 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279006 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480378 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458697 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587458 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03812 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269561 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165193 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282855 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765759 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168791 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802754 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504632 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851205 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498755 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510279 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376997 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609809 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058196 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567277 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568704 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323352 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487171 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.089553 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099588 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663897 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037596 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553673 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409539 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634487 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470391 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477454 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466614 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586217 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270082 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044145 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481342 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648509 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567033 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294216 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315326 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424704 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444786 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370733 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353626 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042987 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316869 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663948 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342617 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480624 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363349 s, 1.4 GB/s , 0.0361957 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651721 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465521 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429672 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624457 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436916 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409037 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449433 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507753 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264848 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437334 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346852 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490572 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585663 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504751 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244884 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399114 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577979 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780871 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048558 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512276 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470346 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0468018 s, 1.1 GB/s , 0.0265057 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044868 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419918 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360982 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781127 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600569 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365163 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357169 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367454 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680236 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363238 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471726 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698523 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043689 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442139 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320975 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059179 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333231 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541613 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563028 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288253 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553875 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05624 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422674 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507938 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059921 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505554 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505545 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261683 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461527 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525434 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422016 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044222 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362844 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427226 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407602 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391311 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457424 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455864 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359889 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641246 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374043 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438491 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486397 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032316 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595603 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335441 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420169 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781918 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380632 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919624 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686543 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677117 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519162 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469677 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630217 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512704 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229982 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562705 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603024 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419803 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378749 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280345 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635412 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486394 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730733 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718682 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395273 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604695 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793463 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737422 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037615 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380785 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657587 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296291 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843565 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024983 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413258 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642272 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700634 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267071 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354495 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278174 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433047 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459442 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405892 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832043 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697589 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738408 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607858 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100347 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666167 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451055 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028993 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743306 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937761 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604422 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039474 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650644 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603072 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422237 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518972 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414392 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366479 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607318 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609266 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537659 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621752 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275069 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510162 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492308 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442842 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397782 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779293 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412532 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929902 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739097 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773355 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354209 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366806 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736994 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530579 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627873 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644993 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140201 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178143 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187726 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778266 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120108 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624854 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304884 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932486 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551914 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408827 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749072 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605758 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813293 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565686 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491113 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535413 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904699 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879505 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103793 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492957 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053969 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500704 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306724 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503789 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284449 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255073 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546135 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569735 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079214 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520181 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849673 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443413 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266427 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845833 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699987 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608022 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692966 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401721 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04872 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419899 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948163 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403047 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571037 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267144 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021007 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771212 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632991 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697895 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632662 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853739 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530027 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178733 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552942 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217303 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541546 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320931 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353218 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323237 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676449 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648215 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704457 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664144 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693485 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295304 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406164 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442879 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529147 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028449 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322592 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543862 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415973 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509476 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574512 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402597 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362243 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470465 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595264 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589215 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543693 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054667 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021546 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853021 s, 615 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701114 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037153 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873893 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488745 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734709 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646006 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041128 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764185 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598845 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413202 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0733877 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706241 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615113 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290179 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369661 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045168 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617016 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616234 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650097 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476544 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555778 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125974 s, 416 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132036 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653049 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715313 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455136 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577228 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433163 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579707 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452026 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287501 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367213 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175809 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628136 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556861 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074497 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817443 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239545 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100758 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184817 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216341 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760809 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320867 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519751 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050822 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375301 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727852 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635447 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722068 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073229 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447366 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665795 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635955 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229707 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693176 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469412 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538274 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272508 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048678 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282484 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669915 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396151 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876999 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482439 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582296 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544608 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023694 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278024 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322784 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337539 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067838 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888278 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931724 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120174 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803446 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953948 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795161 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707151 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788616 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312397 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050531 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517956 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567266 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317732 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618921 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368473 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415159 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503256 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431506 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023124 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471971 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034076 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0977747 s, 536 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540059 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783549 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564097 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476619 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341053 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034135 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327722 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300178 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329233 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702135 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028205 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406538 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352234 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059628 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372913 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036339 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308635 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052114 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043996 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468109 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690366 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320043 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504417 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673058 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611297 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506333 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237798 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234857 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421323 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619664 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544379 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035847 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1022 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615921 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836872 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597305 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655626 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258334 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384016 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517352 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419991 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503234 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894001 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510843 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715051 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570602 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051961 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354952 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334638 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433264 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046751 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532241 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644145 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455309 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223749 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496381 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499697 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418912 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568485 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556397 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654055 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536938 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619252 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053285 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494887 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060437 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378492 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034995 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312129 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315454 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470981 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053959 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322586 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410218 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488911 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355181 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077964 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299016 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020995 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791464 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361469 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518093 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466999 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289425 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323412 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354853 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475541 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634469 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671825 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440779 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449732 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305619 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449105 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367011 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608308 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535599 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761848 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591533 s, 886 MB/s Finish ======== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105612 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766309 s, 684 MB/s detach: must specify one or more FIDs Detach given files from the Persistent Client Cache. usage: lfs pcc detach [-k] FILE [...] lfs commands used to interact with PCC features: lfs pcc attach - attach given files to Persistent Client Cache lfs pcc attach_fid - attach given files into PCC by FID(s) lfs pcc state - display the PCC state for given files lfs pcc detach - detach given files from Persistent Client Cache lfs pcc detach_fid - detach given files from PCC by FID(s) lfs pcc delete - delete the PCC layout componenet for given files lfs pcc pin - pin give files for PCC lfs pcc unpin - unpin given files for PCC llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1073741824 llite.lustre-ffff88012d5b8000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d96.sanity-pcc is idle now PASS 96 (67s) == sanity-pcc test 97: two mmap I/O and layout lock cancel ========================================================== 00:27:55 (1746160075) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.108643 s, 579 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d97.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 490e43d8-0b96-421a-9be9-d0790825791b Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f97.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=490e43d8-0b96-421a-9be9-d0790825791b (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d97.sanity-pcc/d97.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1G llite.lustre-ffff88012d5b8000.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.941185 s, 55.7 MB/s T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1073741824 llite.lustre-ffff88012d5b8000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d97.sanity-pcc is idle now PASS 97 (129s) == sanity-pcc test 98: racer between auto attach and mmap I/O ========================================================== 00:30:04 (1746160204) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.15759 s, 399 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d98.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 1911ab9d-9e21-4fc8-a816-c24168dfc9f0 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f98.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1911ab9d-9e21-4fc8-a816-c24168dfc9f0 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d98.sanity-pcc/d98.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800b5fc9000.pcc_async_threshold=0 llite.lustre-ffff88012d5b8000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.735088 s, 71.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.813131 s, 64.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.826079 s, 63.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.884602 s, 59.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795662 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708693 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381139 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082473 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577167 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530441 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767555 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.45555 s, 36.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033026 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370765 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305818 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791223 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314493 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336824 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638511 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515861 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594114 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666081 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741382 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870302 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10689 s, 490 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900001 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593423 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541651 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627219 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212667 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943342 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0970619 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120588 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886743 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.722055 s, 72.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.741937 s, 70.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.95165 s, 55.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577446 s, 90.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.602951 s, 87.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.520018 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.829117 s, 63.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.61078 s, 85.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.580324 s, 90.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.40109 s, 37.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.46852 s, 35.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.63526 s, 32.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.914133 s, 57.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02021 s, 51.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.06169 s, 49.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196637 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265745 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.667341 s, 78.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.529021 s, 99.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.273004 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.528563 s, 99.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.799889 s, 65.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.632268 s, 82.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165545 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.663798 s, 79.0 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.384838 s, 136 MB/s 52428800 bytes (52 MB) copied, 0.37198 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954197 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.627375 s, 83.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102772 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202218 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.27818 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.422995 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.431156 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1571 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.58048 s, 90.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.470761 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.476347 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489476 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.398315 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.488019 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.476472 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.320241 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.413041 s, 127 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181566 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158556 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159257 s, 329 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.47672 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.47444 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369946 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.490405 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.942862 s, 55.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.078 s, 48.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.52673 s, 34.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.44493 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457793 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.408172 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.418608 s, 125 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.649131 s, 80.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.742844 s, 70.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.843933 s, 62.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.749605 s, 69.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.331959 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892057 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11195 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169055 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513313 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560641 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778553 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336294 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107864 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416736 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043702 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124472 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357691 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416759 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059458 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062502 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717814 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232483 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10882 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371951 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247235 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107181 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512335 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547477 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509493 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613118 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491778 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0928272 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412605 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334993 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.703552 s, 74.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.727092 s, 72.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.742383 s, 70.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.604146 s, 86.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.936765 s, 56.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.867946 s, 60.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.3437 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.46512 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.552539 s, 94.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335435 s, 156 MB/s , 0.39943 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.376149 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.366217 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762591 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.370337 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264386 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209414 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.564884 s, 92.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.356773 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.361196 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.847873 s, 61.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530712 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451762 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447494 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791677 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325099 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334098 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048613 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363778 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039823 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750525 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873287 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582039 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22298 s, 235 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03641 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480298 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652785 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759899 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495338 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536039 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473531 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740408 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452364 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063977 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659536 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705634 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.378934 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.611332 s, 85.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219346 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.310203 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.357249 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.565744 s, 92.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369524 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.36348 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.363 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.659528 s, 79.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.765285 s, 68.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.682935 s, 76.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768705 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.702135 s, 74.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.654427 s, 80.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.512872 s, 102 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.279401 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.559564 s, 93.7 MB/s 52428800 bytes (52 MB) copied, 0.307137 s, 171 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.895237 s, 58.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.384547 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.34786 s, 38.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.26923 s, 41.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.22176 s, 42.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02755 s, 51.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.03695 s, 50.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233711 s, 224 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233 s, 225 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.556317 s, 94.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.913038 s, 57.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.960586 s, 54.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.5042 s, 34.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.522235 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.59313 s, 32.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.63349 s, 32.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.625082 s, 83.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.893605 s, 58.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.807126 s, 65.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14454 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445055 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.11945 s, 46.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.03515 s, 50.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.08069 s, 48.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692737 s, 757 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.87241 s, 60.1 MB/s , 0.871961 s, 60.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.592953 s, 88.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676028 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169356 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265835 s, 197 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.610056 s, 85.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.613718 s, 85.4 MB/s 52428800 bytes (52 MB) copied, 0.886712 s, 59.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23486 s, 223 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.370584 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.810101 s, 64.7 MB/s 52428800 bytes (52 MB) copied, 0.436427 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.461869 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.383541 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.302646 s, 173 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.503316 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.554578 s, 94.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.452321 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.420716 s, 125 MB/s llite.lustre-ffff8800b5fc9000.pcc_async_threshold=1073741824 llite.lustre-ffff88012d5b8000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d98.sanity-pcc is idle now PASS 98 (69s) == sanity-pcc test 99: race among unlink | mmap read | write | detach for PCC-RO file ========================================================== 00:31:13 (1746160273) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.230025 s, 274 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: a4c3a871-d30e-460b-88c9-91d0f674a118 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f99.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a4c3a871-d30e-460b-88c9-91d0f674a118 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.80696 s, 65.0 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.846535 s, 61.9 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00194 s, 52.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218855 s, 2.4 GB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.816019 s, 64.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283195 s, 1.9 GB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.53594 s, 97.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.331836 s, 158 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.330873 s, 158 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.47969 s, 35.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.01955 s, 51.4 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.14769 s, 45.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.557061 s, 94.1 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283942 s, 1.8 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.58321 s, 33.1 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320256 s, 1.6 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.885622 s, 59.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.775012 s, 67.6 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.99768 s, 52.6 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.894138 s, 58.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.08384 s, 48.4 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.33057 s, 39.4 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0944756 s, 555 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.751356 s, 69.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.35624 s, 38.7 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.377707 s, 139 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.598098 s, 87.7 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.81987 s, 63.9 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.34097 s, 39.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259085 s, 2.0 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.01797 s, 51.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.788137 s, 66.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.68374 s, 31.1 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725402 s, 723 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.978804 s, 53.6 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.643884 s, 81.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.19607 s, 43.8 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.965179 s, 54.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640412 s, 819 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.677882 s, 77.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.548581 s, 95.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.943726 s, 55.6 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.785604 s, 66.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348906 s, 1.5 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.288174 s, 182 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.801734 s, 65.4 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.479764 s, 109 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.996313 s, 52.6 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.740838 s, 70.8 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.13452 s, 46.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537359 s, 976 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.892914 s, 58.7 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.314337 s, 167 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.18063 s, 44.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563973 s, 930 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445572 s, 118 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.928091 s, 56.5 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.1229 s, 46.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368395 s, 1.4 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.36562 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.786927 s, 66.6 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.840974 s, 62.3 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.569925 s, 92.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.05086 s, 49.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.797548 s, 65.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383444 s, 1.4 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639034 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.883665 s, 59.3 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.44492 s, 36.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027894 s, 1.9 GB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.947574 s, 55.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.91459 s, 57.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.22892 s, 42.7 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449226 s, 1.2 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.839121 s, 62.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357388 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.687459 s, 76.3 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049781 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.954745 s, 54.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315383 s, 1.7 GB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.389833 s, 134 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.27604 s, 41.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.782776 s, 67.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.983561 s, 53.3 MB/s ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.690284 s, 76.0 MB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800b5fc9000.rpc_stats= snapshot_time: 1746160337.097813698 secs.nsecs start_time: 1746159782.527302279 secs.nsecs elapsed_time: 554.570511419 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 2 2 | 3 7 7 2: 0 0 2 | 0 0 7 4: 0 0 2 | 0 0 7 8: 0 0 2 | 0 0 7 16: 0 0 2 | 0 0 7 32: 0 0 2 | 0 0 7 64: 0 0 2 | 0 0 7 128: 0 0 2 | 0 0 7 256: 12 11 14 | 0 0 7 512: 8 7 21 | 2 4 11 1024: 83 78 100 | 37 88 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 25 23 23 | 16 38 38 2: 14 13 36 | 24 57 95 3: 12 11 48 | 2 4 100 4: 12 11 59 | 0 0 100 5: 9 8 67 | 0 0 100 6: 10 9 77 | 0 0 100 7: 7 6 83 | 0 0 100 8: 7 6 90 | 0 0 100 9: 10 9 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 11 11 | 6 14 14 1: 0 0 11 | 0 0 14 2: 0 0 11 | 0 0 14 4: 0 0 11 | 0 0 14 8: 0 0 11 | 0 0 14 16: 0 0 11 | 0 0 14 32: 0 0 11 | 0 0 14 64: 0 0 11 | 0 0 14 128: 0 0 11 | 0 0 14 256: 8 7 18 | 0 0 14 512: 1 0 19 | 0 0 14 1024: 7 6 26 | 4 9 23 2048: 15 14 40 | 6 14 38 4096: 28 26 66 | 12 28 66 8192: 35 33 100 | 14 33 100 osc.lustre-OST0000-osc-ffff88012d5b8000.rpc_stats= snapshot_time: 1746160337.097958543 secs.nsecs start_time: 1746159784.542055254 secs.nsecs elapsed_time: 552.555903289 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 1 7 7 | 0 0 0 512: 1 7 14 | 0 0 0 1024: 12 85 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 5 35 35 | 0 0 0 2: 3 21 57 | 0 0 0 3: 3 21 78 | 0 0 0 4: 2 14 92 | 0 0 0 5: 1 7 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 7 7 | 0 0 0 1: 0 0 7 | 0 0 0 2: 0 0 7 | 0 0 0 4: 0 0 7 | 0 0 0 8: 0 0 7 | 0 0 0 16: 0 0 7 | 0 0 0 32: 0 0 7 | 0 0 0 64: 0 0 7 | 0 0 0 128: 0 0 7 | 0 0 0 256: 1 7 14 | 0 0 0 512: 0 0 14 | 0 0 0 1024: 1 7 21 | 0 0 0 2048: 2 14 35 | 0 0 0 4096: 4 28 64 | 0 0 0 8192: 5 35 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800b5fc9000.rpc_stats= snapshot_time: 1746160337.098019790 secs.nsecs start_time: 1746159782.536692291 secs.nsecs elapsed_time: 554.561327499 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 512 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 701 10 10 | 3 0 0 2: 9 0 10 | 0 0 0 4: 5 0 10 | 0 0 0 8: 5 0 10 | 0 0 0 16: 2786 39 50 | 0 0 0 32: 193 2 52 | 0 0 0 64: 117 1 54 | 0 0 0 128: 311 4 58 | 0 0 0 256: 631 9 68 | 4 0 1 512: 275 3 71 | 30 5 6 1024: 1962 28 100 | 562 93 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4192 59 59 | 247 41 41 2: 1350 19 79 | 284 47 88 3: 517 7 86 | 57 9 98 4: 337 4 91 | 10 1 99 5: 217 3 94 | 1 0 100 6: 133 1 96 | 0 0 100 7: 107 1 97 | 0 0 100 8: 69 0 98 | 0 0 100 9: 73 1 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 182 2 2 | 49 8 8 1: 77 1 3 | 0 0 8 2: 150 2 5 | 0 0 8 4: 6 0 5 | 0 0 8 8: 2 0 5 | 0 0 8 16: 144 2 8 | 0 0 8 32: 254 3 11 | 0 0 8 64: 420 6 17 | 0 0 8 128: 815 11 29 | 0 0 8 256: 414 5 35 | 2 0 8 512: 208 2 38 | 3 0 9 1024: 701 10 48 | 48 8 17 2048: 1001 14 62 | 93 15 32 4096: 1219 17 79 | 188 31 63 8192: 1402 20 100 | 216 36 100 osc.lustre-OST0001-osc-ffff88012d5b8000.rpc_stats= snapshot_time: 1746160337.098076776 secs.nsecs start_time: 1746159784.542557980 secs.nsecs elapsed_time: 552.555518796 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 1 7 7 | 0 0 0 512: 1 7 14 | 0 0 0 1024: 12 85 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4 28 28 | 0 0 0 2: 2 14 42 | 0 0 0 3: 2 14 57 | 0 0 0 4: 2 14 71 | 0 0 0 5: 1 7 78 | 0 0 0 6: 1 7 85 | 0 0 0 7: 1 7 92 | 0 0 0 8: 1 7 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 7 7 | 0 0 0 1: 0 0 7 | 0 0 0 2: 0 0 7 | 0 0 0 4: 0 0 7 | 0 0 0 8: 0 0 7 | 0 0 0 16: 0 0 7 | 0 0 0 32: 0 0 7 | 0 0 0 64: 0 0 7 | 0 0 0 128: 0 0 7 | 0 0 0 256: 1 7 14 | 0 0 0 512: 0 0 14 | 0 0 0 1024: 1 7 21 | 0 0 0 2048: 2 14 35 | 0 0 0 4096: 4 28 64 | 0 0 0 8192: 5 35 100 | 0 0 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99.sanity-pcc is idle now PASS 99 (67s) == sanity-pcc test 99b: race among unlink | two readers | write | detach for PCC-RO file ========================================================== 00:32:20 (1746160340) 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.436911 s, 480 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: dec9aeae-dcdd-4243-aa39-74e7e86df5ad Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f99b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=dec9aeae-dcdd-4243-aa39-74e7e86df5ad (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.684791 s, 76.6 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.867621 s, 60.4 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.39761 s, 37.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.883977 s, 59.3 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.936094 s, 56.0 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.802854 s, 65.3 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.892386 s, 58.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.652448 s, 80.4 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863916 s, 607 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.38271 s, 37.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.521834 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745649 s, 703 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.36581 s, 38.4 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.625151 s, 83.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421296 s, 1.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.04274 s, 50.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.495917 s, 106 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588703 s, 891 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.821608 s, 63.8 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.436217 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373486 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.757743 s, 69.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.462063 s, 113 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.414389 s, 127 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.26299 s, 41.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415179 s, 126 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445383 s, 118 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.29935 s, 40.4 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.649364 s, 80.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143658 s, 365 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.726878 s, 72.1 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397283 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411722 s, 1.3 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352018 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.993491 s, 52.8 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246401 s, 213 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.290379 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386956 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.836575 s, 62.7 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410565 s, 128 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430796 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.816357 s, 64.2 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348265 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133583 s, 392 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618012 s, 848 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.691234 s, 75.8 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.433759 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.45254 s, 116 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.438954 s, 119 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.38608 s, 37.8 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686913 s, 763 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.451087 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.494528 s, 106 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.800256 s, 65.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.548203 s, 95.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.730907 s, 71.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269274 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.791715 s, 66.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384913 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.722926 s, 72.5 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.622174 s, 84.3 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529996 s, 989 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04503 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.14829 s, 45.7 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.344772 s, 152 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137315 s, 382 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.755737 s, 69.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.448584 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497255 s, 1.1 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.694666 s, 75.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.407503 s, 129 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528823 s, 991 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.89284 s, 58.7 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.521811 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379422 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.916224 s, 57.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.429726 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383963 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.780195 s, 67.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.446243 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394245 s, 1.3 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.873836 s, 60.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.357676 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279323 s, 1.9 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.772627 s, 67.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.41369 s, 127 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576469 s, 909 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.06705 s, 49.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.464994 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471625 s, 1.1 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.761151 s, 68.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.540043 s, 97.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368215 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.901016 s, 58.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.511208 s, 103 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.507948 s, 103 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.988806 s, 53.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.632988 s, 82.8 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497912 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.99794 s, 52.5 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.365656 s, 143 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414029 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.875215 s, 59.9 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360595 s, 1.5 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.475714 s, 110 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18504 s, 283 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.07932 s, 48.6 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243713 s, 215 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.390426 s, 134 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456009 s, 1.1 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.887009 s, 59.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.484586 s, 108 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.272636 s, 192 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127181 s, 412 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02618 s, 51.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.45542 s, 115 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285082 s, 1.8 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.907617 s, 57.8 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.531229 s, 98.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349247 s, 1.5 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.289199 s, 181 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.05814 s, 49.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719349 s, 729 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.670649 s, 78.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.952303 s, 55.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476305 s, 1.1 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59dc/0x2000059f1:0x2d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.442259 s, 119 MB/s ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.769623 s, 68.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463609 s, 1.1 GB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800b5fc9000.rpc_stats= snapshot_time: 1746160404.010804789 secs.nsecs start_time: 1746159782.527302279 secs.nsecs elapsed_time: 621.483502510 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 2 2 | 3 7 7 2: 0 0 2 | 0 0 7 4: 0 0 2 | 0 0 7 8: 0 0 2 | 0 0 7 16: 0 0 2 | 0 0 7 32: 0 0 2 | 0 0 7 64: 0 0 2 | 0 0 7 128: 0 0 2 | 0 0 7 256: 12 11 14 | 0 0 7 512: 8 7 21 | 2 4 11 1024: 83 78 100 | 37 88 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 25 23 23 | 16 38 38 2: 14 13 36 | 24 57 95 3: 12 11 48 | 2 4 100 4: 12 11 59 | 0 0 100 5: 9 8 67 | 0 0 100 6: 10 9 77 | 0 0 100 7: 7 6 83 | 0 0 100 8: 7 6 90 | 0 0 100 9: 10 9 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 11 11 | 6 14 14 1: 0 0 11 | 0 0 14 2: 0 0 11 | 0 0 14 4: 0 0 11 | 0 0 14 8: 0 0 11 | 0 0 14 16: 0 0 11 | 0 0 14 32: 0 0 11 | 0 0 14 64: 0 0 11 | 0 0 14 128: 0 0 11 | 0 0 14 256: 8 7 18 | 0 0 14 512: 1 0 19 | 0 0 14 1024: 7 6 26 | 4 9 23 2048: 15 14 40 | 6 14 38 4096: 28 26 66 | 12 28 66 8192: 35 33 100 | 14 33 100 osc.lustre-OST0000-osc-ffff88012d5b8000.rpc_stats= snapshot_time: 1746160404.013574323 secs.nsecs start_time: 1746159784.542055254 secs.nsecs elapsed_time: 619.471519069 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 1 7 7 | 0 0 0 512: 1 7 14 | 0 0 0 1024: 12 85 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 5 35 35 | 0 0 0 2: 3 21 57 | 0 0 0 3: 3 21 78 | 0 0 0 4: 2 14 92 | 0 0 0 5: 1 7 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 7 7 | 0 0 0 1: 0 0 7 | 0 0 0 2: 0 0 7 | 0 0 0 4: 0 0 7 | 0 0 0 8: 0 0 7 | 0 0 0 16: 0 0 7 | 0 0 0 32: 0 0 7 | 0 0 0 64: 0 0 7 | 0 0 0 128: 0 0 7 | 0 0 0 256: 1 7 14 | 0 0 0 512: 0 0 14 | 0 0 0 1024: 1 7 21 | 0 0 0 2048: 2 14 35 | 0 0 0 4096: 4 28 64 | 0 0 0 8192: 5 35 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800b5fc9000.rpc_stats= snapshot_time: 1746160404.013647586 secs.nsecs start_time: 1746159782.536692291 secs.nsecs elapsed_time: 621.476955295 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 512 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 701 8 8 | 3 0 0 2: 9 0 8 | 0 0 0 4: 5 0 9 | 0 0 0 8: 5 0 9 | 0 0 0 16: 2786 35 44 | 0 0 0 32: 193 2 46 | 0 0 0 64: 117 1 48 | 0 0 0 128: 311 3 52 | 0 0 0 256: 787 9 62 | 12 1 1 512: 355 4 66 | 66 5 7 1024: 2624 33 100 | 1061 92 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4504 57 57 | 448 39 39 2: 1519 19 76 | 570 49 89 3: 646 8 84 | 97 8 97 4: 426 5 89 | 26 2 99 5: 273 3 93 | 1 0 100 6: 175 2 95 | 0 0 100 7: 142 1 97 | 0 0 100 8: 98 1 98 | 0 0 100 9: 109 1 99 | 0 0 100 10: 1 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 255 3 3 | 90 7 7 1: 77 0 4 | 0 0 7 2: 150 1 6 | 0 0 7 4: 6 0 6 | 0 0 7 8: 2 0 6 | 0 0 7 16: 144 1 8 | 0 0 7 32: 254 3 11 | 0 0 7 64: 420 5 16 | 0 0 7 128: 815 10 26 | 0 0 7 256: 480 6 32 | 3 0 8 512: 230 2 35 | 4 0 8 1024: 764 9 45 | 92 8 16 2048: 1128 14 59 | 181 15 32 4096: 1451 18 78 | 362 31 64 8192: 1717 21 100 | 410 35 100 osc.lustre-OST0001-osc-ffff88012d5b8000.rpc_stats= snapshot_time: 1746160404.013699460 secs.nsecs start_time: 1746159784.542557980 secs.nsecs elapsed_time: 619.471141480 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 1 7 7 | 0 0 0 512: 1 7 14 | 0 0 0 1024: 12 85 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4 28 28 | 0 0 0 2: 2 14 42 | 0 0 0 3: 2 14 57 | 0 0 0 4: 2 14 71 | 0 0 0 5: 1 7 78 | 0 0 0 6: 1 7 85 | 0 0 0 7: 1 7 92 | 0 0 0 8: 1 7 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 7 7 | 0 0 0 1: 0 0 7 | 0 0 0 2: 0 0 7 | 0 0 0 4: 0 0 7 | 0 0 0 8: 0 0 7 | 0 0 0 16: 0 0 7 | 0 0 0 32: 0 0 7 | 0 0 0 64: 0 0 7 | 0 0 0 128: 0 0 7 | 0 0 0 256: 1 7 14 | 0 0 0 512: 0 0 14 | 0 0 0 1024: 1 7 21 | 0 0 0 2048: 2 14 35 | 0 0 0 4096: 4 28 64 | 0 0 0 8192: 5 35 100 | 0 0 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99b.sanity-pcc is idle now PASS 99b (67s) == sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 00:33:27 (1746160407) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.112158 s, 561 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 18d1abb6-55eb-4eba-a29b-4898196ff58a Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=18d1abb6-55eb-4eba-a29b-4898196ff58a (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.618477 s, 84.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.56496 s, 5.6 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.025694 s, 40.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.725637 s, 72.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.540984 s, 96.9 MB/s 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.301961 s, 108 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0103805 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.419704 s, 125 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.599767 s, 87.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00881 s, 52.0 MB/s dd: warning: partial read (274432 bytes); suggest iflag=fullblock 0+4 records in 0+4 records out 1048576 bytes (1.0 MB) copied, 0.0213925 s, 49.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 13+0 records in 13+0 records out 13631488 bytes (14 MB) copied, 0.180184 s, 75.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 21+0 records in 21+0 records out 22020096 bytes (22 MB) copied, 0.062609 s, 352 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.943218 s, 55.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602677 s, 870 MB/s 14+0 records in 14+0 records out 14680064 bytes (15 MB) copied, 0.048842 s, 301 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.58415 s, 89.8 MB/s 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.198135 s, 26.5 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00998602 s, 105 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.669264 s, 78.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401366 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397177 s, 1.3 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0464229 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.495447 s, 106 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224048 s, 234 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.857421 s, 61.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.614395 s, 85.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.345998 s, 152 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.174111 s, 193 MB/s 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.201736 s, 187 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.648543 s, 80.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.229385 s, 229 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233844 s, 224 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 dd: warning: partial read (307200 bytes); suggest iflag=fullblock 0+3 records in 0+3 records out 1048576 bytes (1.0 MB) copied, 0.0219431 s, 47.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 18+0 records in 18+0 records out 18874368 bytes (19 MB) copied, 0.0867223 s, 218 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.643393 s, 81.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275997 s, 1.9 GB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0177229 s, 59.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.649299 s, 80.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.501812 s, 104 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276016 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276345 s, 1.9 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0147718 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory dd: warning: partial read (1024000 bytes); suggest iflag=fullblock 41+2 records in 41+2 records out 44040192 bytes (44 MB) copied, 0.289439 s, 152 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.568503 s, 92.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241902 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027677 s, 1.9 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.051739 s, 0.0 kB/s dd: warning: partial read (286720 bytes); suggest iflag=fullblock 0+3 records in 0+3 records out 1048576 bytes (1.0 MB) copied, 0.0272215 s, 38.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.677214 s, 77.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.328844 s, 159 MB/s 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.123154 s, 392 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0261839 s, 0.0 kB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.66237 s, 79.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.349272 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02262 s, 2.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397474 s, 132 MB/s 33+0 records in 33+0 records out 34603008 bytes (35 MB) copied, 0.198932 s, 174 MB/s 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.243285 s, 207 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515861 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.26079 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.916822 s, 57.2 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.374409 s, 5.6 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0112767 s, 93.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.686131 s, 76.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359045 s, 146 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311014 s, 1.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455589 s, 1.2 GB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0502385 s, 20.9 MB/s 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0245491 s, 427 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.254264 s, 206 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.610401 s, 85.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.14673 s, 45.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423112 s, 1.2 GB/s 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.193237 s, 147 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory dd: warning: partial read (667648 bytes); suggest iflag=fullblock 41+3 records in 41+3 records out 44040192 bytes (44 MB) copied, 0.297702 s, 148 MB/s dd: warning: partial read (57344 bytes); suggest iflag=fullblock 43+7 records in 43+7 records out 46047232 bytes (46 MB) copied, 0.289383 s, 159 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.766533 s, 68.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 39+0 records in 39+0 records out 40894464 bytes (41 MB) copied, 0.135855 s, 301 MB/s 22+0 records in 22+0 records out 23068672 bytes (23 MB) copied, 0.136327 s, 169 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477851 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271652 s, 193 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.88327 s, 59.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.176665 s, 11.9 MB/s 20+0 records in 20+0 records out 20971520 bytes (21 MB) copied, 0.200615 s, 105 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 16+0 records in 16+0 records out 16777216 bytes (17 MB) copied, 0.132442 s, 127 MB/s 20+0 records in 20+0 records out 20971520 bytes (21 MB) copied, 0.0838295 s, 250 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.19532 s, 43.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.487891 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218034 s, 240 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.232674 s, 122 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 45+0 records in 45+0 records out 47185920 bytes (47 MB) copied, 0.253839 s, 186 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277364 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.336877 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.838881 s, 62.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312532 s, 1.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 38+0 records in 38+0 records out 39845888 bytes (40 MB) copied, 0.208965 s, 191 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... dd: warning: partial read (217088 bytes); suggest iflag=fullblock 39+4 records in 39+4 records out 41943040 bytes (42 MB) copied, 0.316158 s, 133 MB/s 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.312768 s, 134 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.784261 s, 66.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 23+1 records in 23+1 records out 24621056 bytes (25 MB) copied, 0.736825 s, 33.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0108093 s, 388 MB/s 38+0 records in 38+0 records out 39845888 bytes (40 MB) copied, 0.0488379 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.586903 s, 89.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189553 s, 2.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538526 s, 974 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00989815 s, 0.0 kB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.00921198 s, 455 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.815204 s, 64.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.373886 s, 140 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0567608 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.68902 s, 76.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.485111 s, 108 MB/s 42+0 records in 42+0 records out 44040192 bytes (44 MB) copied, 0.196146 s, 225 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0170821 s, 430 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.720011 s, 72.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.344075 s, 152 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0290072 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00377677 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.312354 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.77052 s, 68.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 20+0 records in 20+0 records out 20971520 bytes (21 MB) copied, 0.256141 s, 81.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.759261 s, 69.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367081 s, 143 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0192662 s, 272 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.596106 s, 88.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262215 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253291 s, 2.1 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.00988538 s, 1.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.623042 s, 84.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.3229 s, 162 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414818 s, 1.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.136751 s, 53.7 MB/s 35+0 records in 35+0 records out 36700160 bytes (37 MB) copied, 0.170046 s, 216 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.400478 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.310639 s, 169 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.867498 s, 60.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 20+0 records in 20+0 records out 20971520 bytes (21 MB) copied, 0.390377 s, 53.7 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.113489 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22001 s, 238 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.226045 s, 232 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.616102 s, 85.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 37+0 records in 37+0 records out 38797312 bytes (39 MB) copied, 0.18109 s, 214 MB/s 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.194173 s, 124 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.739443 s, 70.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.213513 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489193 s, 107 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259368 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308325 s, 1.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 49+1 records in 49+1 records out 51724288 bytes (52 MB) copied, 0.365554 s, 141 MB/s 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.242318 s, 212 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.794505 s, 66.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124147 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.261554 s, 200 MB/s Detach /mnt/lustre/f100.sanity-pcc ... dd: warning: partial read (217088 bytes); suggest iflag=fullblock 23+4 records in 23+4 records out 25165824 bytes (25 MB) copied, 0.140989 s, 178 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.8624 s, 60.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543636 s, 96.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23488 s, 223 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... dd: warning: partial read (630784 bytes); suggest iflag=fullblock 6+3 records in 6+3 records out 7340032 bytes (7.3 MB) copied, 0.112962 s, 65.0 MB/s 14+0 records in 14+0 records out 14680064 bytes (15 MB) copied, 0.026185 s, 561 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.600612 s, 87.3 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0557536 s, 18.8 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0126716 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.61337 s, 85.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.358611 s, 146 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332856 s, 1.6 GB/s dd: warning: partial read (176128 bytes); suggest iflag=fullblock 0+4 records in 0+4 records out 1048576 bytes (1.0 MB) copied, 0.00818871 s, 128 MB/s dd: warning: partial read (602112 bytes); suggest iflag=fullblock 6+3 records in 6+3 records out 7340032 bytes (7.3 MB) copied, 0.0262017 s, 280 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194119 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.565499 s, 92.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.0483 s, 50.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225932 s, 2.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.125035 s, 285 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.47081 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21915 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.820774 s, 63.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307672 s, 1.7 GB/s 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.115077 s, 292 MB/s 22+0 records in 22+0 records out 23068672 bytes (23 MB) copied, 0.0976976 s, 236 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.708268 s, 74.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.299178 s, 175 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026076 s, 2.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0335782 s, 312 MB/s 17+0 records in 17+0 records out 17825792 bytes (18 MB) copied, 0.0179208 s, 995 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.857607 s, 61.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456871 s, 115 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.256568 s, 32.7 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0148815 s, 70.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.709092 s, 73.9 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now PASS 100 (66s) == sanity-pcc test 101a: Test auto attach in mount namespace (simulated container) ========================================================== 00:34:33 (1746160473) creating user namespace for 500 sanityu+ 14950 0.0 0.0 108060 364 pts/0 S+ 00:34 0:00 sleep 600 root 14952 0.0 0.0 114880 984 pts/0 S+ 00:34 0:00 grep sleep Created NS: child (sleep) pid=14950 in 0 seconds running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newuidmap] [14950] [0] [500] [1] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newgidmap] [14950] [0] [500] [1] 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105166 s, 499 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d101a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9a15177b-1f57-4932-a546-e193949a766b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f101a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9a15177b-1f57-4932-a546-e193949a766b (extents) (64bit) (large files) (huge files) Check for temp file in PCC mount Check for temp file in PCC mount from inside namespace Starting copytool 'agt1' on 'oleg217-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d101a.sanity-pcc/d101a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach from inside mount namespace 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.040929 s, 25.0 kB/s file: /mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc, type: readwrite, PCC_file: /d101a.sanity-pcc/59a8/0x2000059f1:0x59:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012d5b8000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012d5b8000.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00131766 s, 777 kB/s Verify auto attach at open from inside NS for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012d5b8000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff8800b5fc9000.lru_size=clear ldlm.namespaces.lustre-MDT0001-mdc-ffff88012d5b8000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d101a.sanity-pcc is idle now PASS 101a (7s) == sanity-pcc test 102: PCC-RO should not hange for io_uring I/O engine ========================================================== 00:34:40 (1746160480) SKIP: sanity-pcc test_102 Client OS does not support io_uring I/O engine SKIP 102 (2s) == sanity-pcc test 203: Verify attach/hit bytes statistics data ========================================================== 00:34:42 (1746160482) 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0241298 s, 435 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d203.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/10240 done Creating filesystem with 10240 1k blocks and 2560 inodes Filesystem UUID: ad1eb98e-b7cf-4c3e-b7ad-49028a1df613 Superblock backups stored on blocks: 8193 Allocating group tables: 0/2 done Writing inode tables: 0/2 done Creating journal (2048 blocks): done Writing superblocks and filesystem accounting information: 0/2 done /tmp/f203.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ad1eb98e-b7cf-4c3e-b7ad-49028a1df613 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d203.sanity-pcc/d203.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.010777 s, 95.0 kB/s ==== stats ==== llite.lustre-ffff8800b5fc9000.stats= snapshot_time 1746160483.981310705 secs.nsecs start_time 1746160483.900833115 secs.nsecs elapsed_time 0.080477590 secs.nsecs read_bytes 1 samples [bytes] 1024 1024 1024 1048576 write_bytes 1 samples [bytes] 1024 1024 1024 1048576 read 3 samples [usecs] 153 2874 3191 8310181 write 1 samples [usecs] 4155 4155 4155 17264025 open 3 samples [usecs] 24 15480 15533 239631817 close 3 samples [usecs] 2411 5873 12793 60636131 getattr 2 samples [usecs] 64 2469 2533 6100057 mknod 1 samples [usecs] 10987 10987 10987 120714169 inode_permission 7 samples [usecs] 2 4757 5027 22655331 opencount 3 samples [reqs] 1 3 6 14 openclosetime 2 samples [usecs] 10956 16338 27294 386964180 pcc_attach 1 samples [reqs] pcc_hit_bytes 2 samples [bytes] 1024 1024 2048 2097152 pcc_attach_bytes 1 samples [bytes] 1024 1024 1024 1048576 llite.lustre-ffff88012d5b8000.stats= snapshot_time 1746160483.981946569 secs.nsecs start_time 1746160483.900855579 secs.nsecs elapsed_time 0.081090990 secs.nsecs attach_num: 1, attach_bytes: 1024, hit_num: 2, hit_bytes: 2048 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d203.sanity-pcc is idle now PASS 203 (4s) == sanity-pcc test 204a: pin/unpin pcc flag ============== 00:34:47 (1746160487) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833371 s, 629 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a919ecab-0e8b-4926-a8ea-a263d8303520 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f204a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a919ecab-0e8b-4926-a8ea-a263d8303520 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204a.sanity-pcc is idle now PASS 204a (5s) == sanity-pcc test 204b: Permission check for the pin/unpin operation ========================================================== 00:34:52 (1746160492) mdt.lustre-MDT0000.enable_pin_gid=0 mdt.lustre-MDT0001.enable_pin_gid=0 /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh: illegal option -- p running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc trusted.pin="[hsm: 100]" running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] unpin: cannot unpin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted mdt.lustre-MDT0000.enable_pin_gid=-1 mdt.lustre-MDT0001.enable_pin_gid=-1 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=500 mdt.lustre-MDT0001.enable_pin_gid=500 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=0 mdt.lustre-MDT0001.enable_pin_gid=0 PASS 204b (8s) == sanity-pcc test 204c: PCC pin/unpin without attach ID specified ========================================================== 00:35:00 (1746160500) pin: failed to get id for '/mnt/lustre/f204c.sanity-pcc': No such file or directory 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.101105 s, 622 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=25,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11153) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg217-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg217-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51304/testresults/sanity-pcc-ldiskfs-DNE-centos7_x86_64-centos7_x86_64-retry1 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.17,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.117@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.117@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: b3eec319-60a3-4022-a107-7c0cb4c88c63 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f204c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b3eec319-60a3-4022-a107-7c0cb4c88c63 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204c.sanity-pcc trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204c.sanity-pcc is idle now PASS 204c (5s) == sanity-pcc test complete, duration 2349 sec =========== 00:35:06 (1746160506) === sanity-pcc: start cleanup 00:35:06 (1746160506) === Stopping clients: oleg217-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg217-client.virtnet /mnt/lustre2 opts: === sanity-pcc: finish cleanup 00:35:12 (1746160512) ===