-----============= acceptance-small: sanity-pcc ============----- Thu May 1 16:23:28 EDT 2025 mgs: CentOS Linux release 7.9.2009 (Core) MGS_OS_VERSION_ID=7 MGS_OS_ID=centos MGS_OS_VERSION_CODE=117440512 MGS_OS_ID_LIKE=rhel fedora centos mds1: CentOS Linux release 7.9.2009 (Core) MDS1_OS_ID_LIKE=rhel fedora centos MDS1_OS_ID=centos MDS1_OS_VERSION_ID=7 MDS1_OS_VERSION_CODE=117440512 ost1: CentOS Linux release 7.9.2009 (Core) OST1_OS_VERSION_CODE=117440512 OST1_OS_VERSION_ID=7 OST1_OS_ID_LIKE=rhel fedora centos OST1_OS_ID=centos client: CentOS Linux release 7.9.2009 (Core) CLIENT_OS_ID=centos CLIENT_OS_ID_LIKE=rhel fedora centos CLIENT_OS_VERSION_ID=7 CLIENT_OS_VERSION_CODE=117440512 === sanity-pcc: start setup 16:23:37 (1746131017) === Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) oleg353-client.virtnet: executing check_config_client /mnt/lustre oleg353-client.virtnet: Checking config lustre mounted on /mnt/lustre Checking servers environments Checking clients oleg353-client.virtnet environments Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b39c1800.idle_timeout=debug osc.lustre-OST0000-osc-ffff880136816000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b39c1800.idle_timeout=debug osc.lustre-OST0001-osc-ffff880136816000.idle_timeout=debug disable quota as required oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all === sanity-pcc: finish setup 16:23:52 (1746131032) === running as uid/gid/euid/egid 500/500/500/500, groups: 500 [true] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d0_runas_test/f7030] nobody:x:99: oleg353-server: ls: cannot access /home/green/git/lustre-release/lustre/tests/except/sanity-pcc.*ex: No such file or directory excepting tests: Killing existing copytools on oleg353-client.virtnet pkill: pidfile not valid Try `pkill --help' for more information. Set HSM on and start Set sanity-hsm HSM policy == sanity-pcc test 1a: Test manual lfs pcc attach with manual HSM restore ========================================================== 16:24:17 (1746131057) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881334 s, 595 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5218ce41-11bc-4e5c-95d5-aacb4b02d947 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5218ce41-11bc-4e5c-95d5-aacb4b02d947 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1a.sanity-pcc/d1a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0455629 s, 168 MB/s Restore testing... Attach and detach testing Start to detach the /mnt/lustre/d1a.sanity-pcc/f1a.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1a.sanity-pcc is idle now PASS 1a (9s) == sanity-pcc test 1b: Test manual lfs pcc attach with restore on remote access ========================================================== 16:24:27 (1746131067) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880144 s, 596 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8c3baec8-975c-435b-8cce-51c2c25ed32c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8c3baec8-975c-435b-8cce-51c2c25ed32c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1b.sanity-pcc/d1b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.052223 s, 147 MB/s Restore testing... Waiting 200s for 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1b.sanity-pcc/f1b.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1b.sanity-pcc is idle now PASS 1b (11s) == sanity-pcc test 1c: Test automated attach using Project ID with manual HSM restore ========================================================== 16:24:38 (1746131078) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b589e000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b62f1000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b589e000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b62f1000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0897016 s, 584 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8f72753a-4ba0-4781-aa78-99d98aaeea3f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8f72753a-4ba0-4781-aa78-99d98aaeea3f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1c.sanity-pcc/d1c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0569746 s, 134 MB/s Restore testing... Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1c.sanity-pcc/f1c.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1c.sanity-pcc is idle now PASS 1c (76s) == sanity-pcc test 1d: Test Project ID with remote access ========================================================== 16:25:54 (1746131154) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a76cc800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b6d25800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a76cc800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6d25800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0970409 s, 540 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: dce52f94-03f0-4d21-8d5c-0de3cb587e11 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=dce52f94-03f0-4d21-8d5c-0de3cb587e11 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1d.sanity-pcc/d1d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0534761 s, 143 MB/s Restore testing... Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1d.sanity-pcc/f1d.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1d.sanity-pcc is idle now PASS 1d (77s) == sanity-pcc test 1e: Test RW-PCC with non-root user ==== 16:27:11 (1746131231) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_1e/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_1e/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0187928 s, 54.5 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0122279 s, 83.7 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0049311 s, 208 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] touch: cannot touch '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre PASS 1e (9s) == sanity-pcc test 1f: Test auto RW-PCC cache with non-root user ========================================================== 16:27:20 (1746131240) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a952a000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b39c2800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a952a000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b39c2800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873875 s, 600 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: aab04a2d-1080-498c-82cf-48577c5ee87c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=aab04a2d-1080-498c-82cf-48577c5ee87c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0147935 s, 69.2 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00557914 s, 184 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=256] [count=1] 1+0 records in 1+0 records out 256 bytes (256 B) copied, 0.00541729 s, 47.3 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] touch: cannot touch '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1f.sanity-pcc is idle now PASS 1f (75s) == sanity-pcc test 1g: General permission test for RW-PCC ========================================================== 16:28:35 (1746131315) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989352 s, 530 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7023258f-1aaf-415e-a377-eb34e085d3b4 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7023258f-1aaf-415e-a377-eb34e085d3b4 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1g.sanity-pcc/d1g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0112872 s, 90.7 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00308463 s, 332 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] detach: cannot detach '/mnt/lustre/f1g.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] Waiting 200s for 'SUCCEED' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0147562 s, 69.4 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1g.sanity-pcc is idle now PASS 1g (8s) == sanity-pcc test 2a: Test multi open when creating ===== 16:28:43 (1746131323) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions oleg353-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a952f000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012db72800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a952f000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012db72800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106671 s, 492 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e17cb594-b604-48b0-b9c5-046e321555ba Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e17cb594-b604-48b0-b9c5-046e321555ba (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2a.sanity-pcc/multiop vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.24655 client oleg353-client.virtnet multiop_bg started multiop_pid=24665 ---------- 1 root root 14 May 1 16:29 /mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc/2343/0x200002341:0x2:0x0 Stopping multiop_pid=24665 (kill 24665 on oleg353-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d2a.sanity-pcc is idle now PASS 2a (59s) == sanity-pcc test 2b: Test multi remote open when creating ========================================================== 16:29:43 (1746131383) SKIP: sanity-pcc test_2b Need 2 or more clients, have 1 SKIP 2b (2s) == sanity-pcc test 2c: Test multi open on different mount points when creating ========================================================== 16:29:44 (1746131384) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a7487800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012c0fb000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a7487800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012c0fb000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765898 s, 685 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e15faaab-e4ac-4530-883b-ca281093d920 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e15faaab-e4ac-4530-883b-ca281093d920 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2c.sanity-pcc/d2c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2c.sanity-pcc/f2c.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.27836 client oleg353-client.virtnet multiop_bg started multiop_pid=27846 Stopping multiop_pid=27846 (kill 27846 on oleg353-client.virtnet) multiopen_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d2c.sanity-pcc is idle now PASS 2c (60s) == sanity-pcc test 3a: Repeat attach/detach operations === 16:30:44 (1746131444) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_3a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.017048 s, 60.1 kB/s Start to RW-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RW-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Start to RO-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RO-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Cleanup PCC backend on /mnt/lustre PASS 3a (6s) == sanity-pcc test 3b: Repeat attach/detach operations on multiple clients ========================================================== 16:30:50 (1746131450) SKIP: sanity-pcc test_3b Need 3 or more clients, have 1 SKIP 3b (2s) == sanity-pcc test 4: Auto cache test for mmap =========== 16:30:52 (1746131452) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a7483000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b60f6000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a7483000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b60f6000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101331 s, 517 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d4.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3023fa51-18a9-47a5-8e56-763e2ccc4137 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f4.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3023fa51-18a9-47a5-8e56-763e2ccc4137 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d4.sanity-pcc/d4.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' mmap test1: basic mmap operation (PASS, 0.031691s) mmap test2: MAP_PRIVATE not write back (PASS, 0.031525s) mmap test3: concurrent mmap ops on two nodes (PASS, 2.044s) mmap test4: c1 write to f1 from mmapped f2, c2 write to f1 from mmapped f1 (PASS, 4.1111s) mmap test5: read/write file to/from the buffer which mmapped to just this file (PASS, 0.031139s) mmap test6: check mmap write/read content on two nodes (PASS, 1.0589s) mmap test7: file i/o with an unmapped buffer (SKIPPED, 0s) mmap test8: SIGBUS for beyond file size (SKIPPED, 0s) mmap test9: SIGBUS for negative file offset (SKIPPED, 0s) mmap test10: multi-thread mmap access (PASS, 0.09346s) mmap test11: multi-thread shared mmap (PASS, 0.14497s) mmap test10: mtime not change for readonly mmap access (PASS, 1.0197s) ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a7483000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b60f6000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d4.sanity-pcc is idle now PASS 4 (80s) == sanity-pcc test 5: Mmap & cat a RW-PCC cached file ==== 16:32:12 (1746131532) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112637 s, 465 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d5.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 21eac729-20f7-4662-a292-e6fa446b625c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f5.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=21eac729-20f7-4662-a292-e6fa446b625c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d5.sanity-pcc/d5.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d5.sanity-pcc is idle now PASS 5 (7s) == sanity-pcc test 6: Test mmap write on RW-PCC ========== 16:32:19 (1746131539) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997121 s, 526 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d6.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2cc3b755-ff98-45e2-86ab-5a3cae0dadc7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f6.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2cc3b755-ff98-45e2-86ab-5a3cae0dadc7 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d6.sanity-pcc/d6.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d6.sanity-pcc is idle now PASS 6 (7s) == sanity-pcc test 7a: Fake file detached between fault() and page_mkwrite() for RW-PCC ========================================================== 16:32:26 (1746131546) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12437 s, 422 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6e5de559-aac5-44d2-b7e7-ff41d7859499 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6e5de559-aac5-44d2-b7e7-ff41d7859499 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7a.sanity-pcc/d7a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1412 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7a.sanity-pcc is idle now PASS 7a (5s) == sanity-pcc test 7b: Test the race with concurrent mkwrite and detach ========================================================== 16:32:31 (1746131551) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0792903 s, 661 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 872f2cc5-73eb-410f-bf40-8fa46875f33e Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=872f2cc5-73eb-410f-bf40-8fa46875f33e (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7b.sanity-pcc/d7b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1413 fail_val=20 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7b.sanity-pcc is idle now PASS 7b (46s) == sanity-pcc test 8: Test fake -ENOSPC tolerance for RW-PCC ========================================================== 16:33:17 (1746131597) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103769 s, 505 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d8.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b88a89d7-9184-4edb-8671-d4fc2a5e3de9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f8.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b88a89d7-9184-4edb-8671-d4fc2a5e3de9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d8.sanity-pcc/d8.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d8.sanity-pcc is idle now PASS 8 (7s) == sanity-pcc test 9: Test -ENOSPC tolerance on loop PCC device for RW-PCC ========================================================== 16:33:24 (1746131604) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139447 s, 376 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.9a: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9bdf466b-ba15-4808-8836-80cf3a58f88f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f9.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9bdf466b-ba15-4808-8836-80cf3a58f88f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.9a/d9.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.9a/d9.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 2.61494 s, 24.1 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.9a is idle now PASS 9 (7s) == sanity-pcc test 10a: Test RW-PCC with user quota on loop PCC device ========================================================== 16:33:31 (1746131611) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0999081 s, 525 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5737f4cf-a752-4750-97e3-60eae052fd82 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5737f4cf-a752-4750-97e3-60eae052fd82 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.196786 s, 79.9 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.158462 s, 99.3 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x26:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10a.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10a.sanity-pcc/uquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 0.746362 s, 42.1 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10a.sanity-pcc is idle now PASS 10a (7s) == sanity-pcc test 10b: Test RW-PCC with group quota on loop PCC device ========================================================== 16:33:38 (1746131618) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067711 s, 774 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 0bc7669b-7619-4bac-bb53-44ac93291fde Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0bc7669b-7619-4bac-bb53-44ac93291fde (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.212238 s, 74.1 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.251279 s, 62.6 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x2b:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10b.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10b.sanity-pcc/gquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 1.77158 s, 17.8 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10b.sanity-pcc is idle now PASS 10b (8s) == sanity-pcc test 10c: Test RO-PCC with user quota on loop PCC device ========================================================== 16:33:46 (1746131626) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971098 s, 540 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 68e73770-f1cc-4e1e-bfc6-9a0fb2a4f5aa Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=68e73770-f1cc-4e1e-bfc6-9a0fb2a4f5aa (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.216993 s, 72.5 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.216516 s, 72.6 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10c.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10c.sanity-pcc is idle now PASS 10c (7s) == sanity-pcc test 10d: Test RO-PCC with group quota on loop PCC device ========================================================== 16:33:53 (1746131633) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126119 s, 416 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 0e8180ba-26c5-477e-bb26-1bf35ff2642b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0e8180ba-26c5-477e-bb26-1bf35ff2642b (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.25788 s, 61.0 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.190546 s, 82.5 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10d.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10d.sanity-pcc is idle now PASS 10d (8s) == sanity-pcc test 10e: Tolerate -EDQUOT failure when auto PCC-RO attach with user quota ========================================================== 16:34:01 (1746131641) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13782 s, 380 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a6bc8b56-a357-4cf2-81ec-9fb0e8e5d682 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a6bc8b56-a357-4cf2-81ec-9fb0e8e5d682 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10e.sanity-pcc/d10e.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: uid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0385334 s, 54.4 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0141364 s, 148 MB/s /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0807613 s, 64.9 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0458836 s, 114 MB/s file: /mnt/lustre/f10e.sanity-pcc, type: none /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10e.sanity-pcc is idle now PASS 10e (6s) == sanity-pcc test 10f: Tolerate -EDQUOT failure when auto PCC-RO attach with group quota ========================================================== 16:34:07 (1746131647) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12046 s, 435 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2db97e96-a4fe-4e27-9e53-178092207440 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2db97e96-a4fe-4e27-9e53-178092207440 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10f.sanity-pcc/d10f.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: gid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0417975 s, 50.2 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0132581 s, 158 MB/s /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0782234 s, 67.0 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0251279 s, 209 MB/s file: /mnt/lustre/f10f.sanity-pcc, type: none /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10f.sanity-pcc is idle now PASS 10f (6s) == sanity-pcc test 11: Test attach fault injection with simulated PCC file path ========================================================== 16:34:14 (1746131654) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891184 s, 588 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d11.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: feea6de5-0c4e-4373-b6ff-6656d9109304 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f11.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=feea6de5-0c4e-4373-b6ff-6656d9109304 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d11.sanity-pcc/d11.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32d9 Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32db lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3a:0x0]: Permission denied (13) lfs pcc pcc: cannot attach with ID: 2: Permission denied (13) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Permission denied lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3b:0x0]: Invalid argument (22) lfs pcc pcc: cannot attach with ID: 2: Invalid argument (22) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Invalid argument Cleanup PCC backend on /mnt/lustre /mnt/pcc.d11.sanity-pcc is idle now PASS 11 (5s) == sanity-pcc test 12: RW-PCC attach races with concurrent HSM remove ========================================================== 16:34:19 (1746131659) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_12/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1414 fail_val=20 /mnt/lustre/f12.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre PASS 12 (25s) == sanity-pcc test 13a: Test auto RW-PCC create caching for UID/GID rule ========================================================== 16:34:44 (1746131684) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: uid={500} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00585995 s, 175 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00618811 s, 165 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00664127 s, 154 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: gid={500} lfs mkdir: dirstripe error on '/mnt/lustre/d13a.sanity-pcc': stripe already set lfs setdirstripe: cannot create dir '/mnt/lustre/d13a.sanity-pcc': File exists running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00304685 s, 336 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00352884 s, 290 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00473523 s, 216 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre PASS 13a (11s) == sanity-pcc test 13b: Test auto RW-PCC create caching for file name with wildcard ========================================================== 16:34:55 (1746131695) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13b/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13b/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: fname={*.h5 suffix.* Mid*dle} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00410589 s, 249 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/suffix.doc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00404609 s, 253 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/MidPADdle] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0108733 s, 94.2 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/Midpad] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00639325 s, 160 kB/s Cleanup PCC backend on /mnt/lustre PASS 13b (5s) == sanity-pcc test 13c: Check auto RW-PCC create caching for UID/GID/ProjID/fname rule ========================================================== 16:35:00 (1746131700) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a74b9000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800a74bb800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a74b9000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a74bb800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724183 s, 724 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d13c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a6839d16-8929-4210-8842-85081f12a33d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f13c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a6839d16-8929-4210-8842-85081f12a33d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100 200}&fname={*.h5},uid={500}&gid={500} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0146253 s, 70.0 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00616828 s, 166 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0153896 s, 66.5 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00410162 s, 250 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13c.sanity-pcc/ugidcache] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00679862 s, 151 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d13c.sanity-pcc is idle now PASS 13c (55s) == sanity-pcc test 14: Revocation of the layout lock should detach the file automatically ========================================================== 16:35:55 (1746131755) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_14/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear Cleanup PCC backend on /mnt/lustre PASS 14 (6s) == sanity-pcc test 15: Test auto attach at open when file is still valid cached ========================================================== 16:36:01 (1746131761) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818357 s, 641 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d15.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 4262bbca-8cb9-4441-bd71-18b6f37008f1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f15.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4262bbca-8cb9-4441-bd71-18b6f37008f1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d15.sanity-pcc/d15.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach for non-root user running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00723838 s, 141 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] file: /mnt/lustre/d15.sanity-pcc/f15.sanity-pcc, type: readwrite, PCC_file: /d15.sanity-pcc/3aa0/0x200003ab1:0x11:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] Verify auto attach at open for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear Waiting 200s for 'SUCCEED' Verify auto attach at open for RO-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d15.sanity-pcc is idle now PASS 15 (9s) == sanity-pcc test 16: Test detach with different options ========================================================== 16:36:11 (1746131771) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845087 s, 620 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d16.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 14b09c61-c78e-4b9c-b85e-bd24c7edaea3 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f16.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=14b09c61-c78e-4b9c-b85e-bd24c7edaea3 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d16.sanity-pcc/d16.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Test detach for RW-PCC Test for reusing valid PCC cache Test for the default detach Waiting 200s for 'SUCCEED' Test detach for RO-PCC detach_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d16.sanity-pcc is idle now PASS 16 (11s) == sanity-pcc test 17: Test auto attach for layout refresh ========================================================== 16:36:21 (1746131781) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091412 s, 574 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d17.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 49c4074b-34d2-4510-b419-1e454c7c667a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f17.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=49c4074b-34d2-4510-b419-1e454c7c667a (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d17.sanity-pcc/d17.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d17.sanity-pcc/d17.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 94 autocache: projid={100} Verify auto attach during IO for truncate Verify auto attach during IO for read/write multiop /mnt/lustre/f17.sanity-pcc vO_r TMPPIPE=/tmp/multiop_open_wait_pipe.19333 client oleg353-client.virtnet multiop_bg started multiop_pid=19343 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear Stopping multiop_pid=19343 (kill 19343 on oleg353-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d17.sanity-pcc is idle now PASS 17 (15s) == sanity-pcc test 18: Verify size correctness after re-attach the file ========================================================== 16:36:36 (1746131796) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910406 s, 576 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d18.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a63a2140-b78a-4948-a415-67da6443b920 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f18.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a63a2140-b78a-4948-a415-67da6443b920 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d18.sanity-pcc/d18.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d18.sanity-pcc/d18.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.120537 s, 34.8 MB/s file: /mnt/lustre/f18.sanity-pcc, type: readwrite, PCC_file: /d18.sanity-pcc/3aac/0x200003ab1:0x1d:0x0, open_count: 0, flags: 0 file: /mnt/lustre/f18.sanity-pcc, type: none /mnt/lustre/f18.sanity-pcc has size 4194304 OK 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00480462 s, 213 kB/s /mnt/lustre2/f18.sanity-pcc has size 1049600 OK file: /mnt/lustre/f18.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d18.sanity-pcc is idle now PASS 18 (8s) == sanity-pcc test 19: Verify the file re-attach works as expected ========================================================== 16:36:44 (1746131804) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655031 s, 800 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d19.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2e53376e-8970-4bf2-bb85-a59de73ce47d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f19.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2e53376e-8970-4bf2-bb85-a59de73ce47d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d19.sanity-pcc/d19.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d19.sanity-pcc is idle now PASS 19 (6s) == sanity-pcc test 20: Auto attach works after the inode was once evicted from cache ========================================================== 16:36:51 (1746131811) 120+0 records in 120+0 records out 125829120 bytes (126 MB) copied, 0.322078 s, 391 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d20.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/122880 done Creating filesystem with 122880 1k blocks and 30720 inodes Filesystem UUID: c5776b87-ab61-40e0-97e7-be1f29fb7140 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/15 done Writing inode tables: 0/15 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/15 done /tmp/f20.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c5776b87-ab61-40e0-97e7-be1f29fb7140 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d20.sanity-pcc/d20.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d20.sanity-pcc is idle now PASS 20 (8s) == sanity-pcc test 21a: PCC-RO storing as a plain HSM mirror component for plain layout ========================================================== 16:37:00 (1746131820) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122083 s, 429 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5cff42dc-b1a0-4344-9d45-b360879849a1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5cff42dc-b1a0-4344-9d45-b360879849a1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21a.sanity-pcc/d21a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Plain layout info before PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 267 0x10b 0x280000400 FLR layout info after PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC attach /mnt/lustre/f21a.sanity-pcc again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC detach '/mnt/lustre/f21a.sanity-pcc' again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21a.sanity-pcc is idle now PASS 21a (6s) == sanity-pcc test 21b: PCC-RO stroing as a plain HSM mirror component for FLR layouts ========================================================== 16:37:05 (1746131825) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850612 s, 616 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: aabad5c1-4794-42c7-aaeb-2e1b64a5a4bc Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=aabad5c1-4794-42c7-aaeb-2e1b64a5a4bc (extents) (64bit) (large files) (huge files) FLR layout before PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc' again: /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21b.sanity-pcc is idle now PASS 21b (6s) == sanity-pcc test 21c: Verify HSM release works storing PCC-RO as HSM mirror component ========================================================== 16:37:11 (1746131831) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117247 s, 447 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: bfe15efb-546a-42b4-884a-1a43f3edd42b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=bfe15efb-546a-42b4-884a-1a43f3edd42b (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21c.sanity-pcc/d21c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' /mnt/lustre/f21c.sanity-pcc: (0x00000009) exists archived, archive_id:2 /mnt/lustre/f21c.sanity-pcc lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 0 lmm_stripe_offset: 0 lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' /mnt/lustre/f21c.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21c.sanity-pcc is idle now PASS 21c (8s) == sanity-pcc test 21d: Write should invalidate PCC-RO caching ========================================================== 16:37:19 (1746131839) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736897 s, 711 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 650e5bda-0fba-45db-8c73-bcf9abf474fd Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=650e5bda-0fba-45db-8c73-bcf9abf474fd (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21d.sanity-pcc/d21d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21d.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 270 0x10e 0x280000400 PCC-RO attach '/mnt/lustre/f21d.sanity-pcc': /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Write invalidated PCC-RO cache: /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21d.sanity-pcc is idle now PASS 21d (5s) == sanity-pcc test 21e: Truncate should invalidate PCC-RO caching ========================================================== 16:37:24 (1746131844) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134152 s, 391 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 61e96559-3092-489e-a553-b0091f0bab40 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=61e96559-3092-489e-a553-b0091f0bab40 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21e.sanity-pcc/d21e.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21e.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 271 0x10f 0x240000400 PCC-RO attach '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Trucate invalidate PCC-RO file '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21e.sanity-pcc is idle now PASS 21e (6s) == sanity-pcc test 21f: mmap write should invalidate PCC-RO caching ========================================================== 16:37:30 (1746131850) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907057 s, 578 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e056343f-9c50-4483-a198-5087df5fc6c0 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e056343f-9c50-4483-a198-5087df5fc6c0 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21f.sanity-pcc/d21f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21f.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 271 0x10f 0x280000400 PCC-RO attach '/mnt/lustre/f21f.sanity-pcc': /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Mmap write invalidate PCC-RO caching: /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21f.sanity-pcc is idle now PASS 21f (6s) == sanity-pcc test 21g: PCC-RO for file under FLR write pending state ========================================================== 16:37:36 (1746131856) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154191 s, 340 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ddccdf7f-8349-4903-b89a-f49444d3b59c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ddccdf7f-8349-4903-b89a-f49444d3b59c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21g.sanity-pcc/d21g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: wp lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp,pccro lcm_layout_gen: 4 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after Write invalidate '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp lcm_layout_gen: 5 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21g.sanity-pcc is idle now PASS 21g (5s) == sanity-pcc test 21h: Extend mirror once file was PCC-RO cached ========================================================== 16:37:41 (1746131861) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996773 s, 526 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21h.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3588e655-e400-49f7-8a1d-1ed019ab641d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21h.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3588e655-e400-49f7-8a1d-1ed019ab641d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21h.sanity-pcc/d21h.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after extend a mirror: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } file: /mnt/lustre/f21h.sanity-pcc, type: none FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc' again: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21h.sanity-pcc is idle now PASS 21h (6s) == sanity-pcc test 21i: HSM release increase layout gen, should invalidate PCC-RO cache ========================================================== 16:37:47 (1746131867) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068531 s, 765 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21i.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 65bdd34e-d0f8-4a87-a86e-135566d2e802 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21i.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=65bdd34e-d0f8-4a87-a86e-135566d2e802 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21i.sanity-pcc/d21i.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21i.sanity-pcc: (0x00000009) exists archived, archive_id:2 Layout after HSM release /mnt/lustre/f21i.sanity-pcc: /mnt/lustre/f21i.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 0 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC state /mnt/lustre/f21i.sanity-pcc: file: /mnt/lustre/f21i.sanity-pcc, type: none Layout after PCC-RO attach /mnt/lustre/f21i.sanity-pcc again: /mnt/lustre/f21i.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 278 0x116 0x280000400 PCC state: file: /mnt/lustre/f21i.sanity-pcc, type: readonly, PCC_file: /d21i.sanity-pcc/3a86/0x200003ab1:0x37:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21i.sanity-pcc is idle now PASS 21i (7s) == sanity-pcc test 21j: PCC-RO for encrypted file ======== 16:37:54 (1746131874) SKIP: sanity-pcc test_21j client encryption not supported SKIP 21j (1s) == sanity-pcc test 22: Test RO-PCC attach for the HSM released file ========================================================== 16:37:55 (1746131875) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885542 s, 592 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d22.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: aa2b768f-1fe6-4591-b27f-e7d0ebfe2b7d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f22.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=aa2b768f-1fe6-4591-b27f-e7d0ebfe2b7d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d22.sanity-pcc/d22.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 2 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x117:0x0] } - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after HSM release /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 0 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc again: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 0 obdidx objid objid group 0 279 0x117 0x240000400 Layout after PCC-RO detach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 0 obdidx objid objid group 0 279 0x117 0x240000400 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d22.sanity-pcc is idle now PASS 22 (11s) == sanity-pcc test 23: Test write/truncate/mmap-write invalidating RO-PCC caching ========================================================== 16:38:06 (1746131886) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771237 s, 680 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d23.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2af77b7e-8191-46d1-bfb7-d2b34c19d761 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f23.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2af77b7e-8191-46d1-bfb7-d2b34c19d761 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d23.sanity-pcc/d23.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Write should invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Truncate invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Finish trucate operation Mmap write invalidates RO-PCC caching /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 7 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Mmap write /mnt/lustre/f23.sanity-pcc via multiop /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 8 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d23.sanity-pcc is idle now PASS 23 (6s) == sanity-pcc test 24a: Test RO-PCC with non-root user === 16:38:12 (1746131892) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163192 s, 321 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 22451874-aa60-49fc-bd8d-ec7b6009c5e4 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=22451874-aa60-49fc-bd8d-ec7b6009c5e4 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: a0 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00652365 s, 157 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00528842 s, 194 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] touch: cannot touch '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24a.sanity-pcc is idle now PASS 24a (6s) == sanity-pcc test 24b: General permission test for RO-PCC ========================================================== 16:38:18 (1746131898) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952554 s, 550 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b57dee1d-4afb-41c9-a78e-bf958b2b4050 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b57dee1d-4afb-41c9-a78e-bf958b2b4050 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24b.sanity-pcc/d24b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00823316 s, 124 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00317257 s, 323 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] detach: cannot detach '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00349969 s, 293 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24b.sanity-pcc is idle now PASS 24b (5s) == sanity-pcc test 25: Tolerate fake read failure for RO-PCC ========================================================== 16:38:23 (1746131903) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769284 s, 682 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d25.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5cfb5c38-28ba-412f-9716-49a53dfc6ef1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f25.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5cfb5c38-28ba-412f-9716-49a53dfc6ef1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d25.sanity-pcc/d25.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 fail_loc=0 fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d25.sanity-pcc is idle now PASS 25 (5s) == sanity-pcc test 26: Repeat the attach/detach when the file has multiple openers ========================================================== 16:38:28 (1746131908) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071238 s, 736 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d26.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a48ebfca-c4b0-4766-928a-56b6b99f0bad Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f26.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a48ebfca-c4b0-4766-928a-56b6b99f0bad (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d26.sanity-pcc/d26.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2617 client oleg353-client.virtnet multiop_bg started multiop_pid=2627 Stopping multiop_pid=2627 (kill 2627 on oleg353-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2732 client oleg353-client.virtnet multiop_bg started multiop_pid=2742 Stopping multiop_pid=2742 (kill 2742 on oleg353-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2847 client oleg353-client.virtnet multiop_bg started multiop_pid=2857 Stopping multiop_pid=2857 (kill 2857 on oleg353-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2962 client oleg353-client.virtnet multiop_bg started multiop_pid=2972 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=2972 (kill 2972 on oleg353-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.3113 client oleg353-client.virtnet multiop_bg started multiop_pid=3123 Waiting 200s for 'SUCCEED' Stopping multiop_pid=3123 (kill 3123 on oleg353-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d26.sanity-pcc is idle now PASS 26 (27s) == sanity-pcc test 27: Auto attach at open when the file has multiple openers ========================================================== 16:38:55 (1746131935) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695026 s, 754 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d27.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6d1f69bc-a9bd-4041-a6d2-e6bdf1c1a406 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f27.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6d1f69bc-a9bd-4041-a6d2-e6bdf1c1a406 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d27.sanity-pcc/d27.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4028 client oleg353-client.virtnet multiop_bg started multiop_pid=4039 Waiting 200s for 'SUCCEED' Stopping multiop_pid=4039 (kill 4039 on oleg353-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4173 client oleg353-client.virtnet multiop_bg started multiop_pid=4183 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=4183 (kill 4183 on oleg353-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4316 client oleg353-client.virtnet multiop_bg started multiop_pid=4326 Stopping multiop_pid=4326 (kill 4326 on oleg353-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4435 client oleg353-client.virtnet multiop_bg started multiop_pid=4445 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74b9000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a74bb800.lru_size=clear Stopping multiop_pid=4445 (kill 4445 on oleg353-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d27.sanity-pcc is idle now PASS 27 (24s) == sanity-pcc test 28: RW-PCC attach should fail when the file has cluster-wide openers ========================================================== 16:39:20 (1746131960) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849105 s, 617 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d28.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ca955186-e8c6-4461-88d5-2466689d4b7b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f28.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ca955186-e8c6-4461-88d5-2466689d4b7b (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d28.sanity-pcc/d28.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.5304 client oleg353-client.virtnet multiop_bg started multiop_pid=5314 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Stopping multiop_pid=5314 (kill 5314 on oleg353-client.virtnet) multiop /mnt/lustre2/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7030 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Cleanup PCC backend on /mnt/lustre /mnt/pcc.d28.sanity-pcc is idle now PASS 28 (10s) == sanity-pcc test 29a: Auto readonly caching on RO-PCC backend for O_RDONLY open ========================================================== 16:39:29 (1746131969) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800aa2b7800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012a60f000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aa2b7800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a60f000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869722 s, 603 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2a680747-5155-45b5-995b-a40ca6ddc82a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2a680747-5155-45b5-995b-a40ca6ddc82a (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 100 P /mnt/lustre/d29a.sanity-pcc/f29a.sanity-pcc Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29a.sanity-pcc is idle now PASS 29a (74s) == sanity-pcc test 29b: Auto PCC-RO attach in atomic_open ========================================================== 16:40:43 (1746132043) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115754 s, 453 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7cfabdb7-0e18-474b-8baa-1d941df852d2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7cfabdb7-0e18-474b-8baa-1d941df852d2 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d29b.sanity-pcc/d29b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.dat} 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.0386862 s, 106 kB/s 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00986883 s, 415 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00758705 s, 540 kB/s vm.drop_caches = 3 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.0320227 s, 128 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29b.sanity-pcc is idle now PASS 29b (8s) == sanity-pcc test 30: Test lctl pcc del command ========= 16:40:51 (1746132051) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104833 s, 500 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d30.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 41b0b851-2418-456a-b0c7-f09938b020d9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f30.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=41b0b851-2418-456a-b0c7-f09938b020d9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 3e autocache: projid={100} check file 0:'4287' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'4289' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'428b' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 1:'0x200004281:0x6:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4287' Handle the file: 0x200004281:0x6:0x0 Detach and remove the PCC cached file: 0x200004281:0x6:0x0 check file 1:'0x200004281:0x8:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289' Handle the file: 0x200004281:0x8:0x0 Remove non-cached file: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289/0x200004281:0x8:0x0 flags: 1 check file 1:'0x200004281:0xa:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/428b' Handle the file: 0x200004281:0xa:0x0 Detach and remove the PCC cached file: 0x200004281:0xa:0x0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d30.sanity-pcc is idle now PASS 30 (5s) == sanity-pcc test 31: Test lctl pcc del command with --keep option ========================================================== 16:40:56 (1746132056) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0804638 s, 652 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d31.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f3e89d00-cc13-478d-917d-cc916e73ff01 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f31.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f3e89d00-cc13-478d-917d-cc916e73ff01 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d31.sanity-pcc/d31.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d31.sanity-pcc/d31.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 30 autocache: projid={100} Cleanup PCC backend on /mnt/lustre /mnt/pcc.d31.sanity-pcc is idle now PASS 31 (7s) == sanity-pcc test 32: Test for RO-PCC when PCC copy is deleted ========================================================== 16:41:03 (1746132063) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728216 s, 720 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d32.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1b60156f-1b8d-4b84-b9ae-6b844f30b409 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f32.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1b60156f-1b8d-4b84-b9ae-6b844f30b409 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d32.sanity-pcc/d32.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f32.sanity-pcc vo_rc TMPPIPE=/tmp/multiop_open_wait_pipe.11602 client oleg353-client.virtnet multiop_bg started multiop_pid=11612 Stopping multiop_pid=11612 (kill 11612 on oleg353-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d32.sanity-pcc is idle now PASS 32 (12s) == sanity-pcc test 33: Cache rule with comparator (>, =, <) for file size ========================================================== 16:41:15 (1746132075) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107667 s, 487 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d33.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7a103274-02b2-436e-8e59-a0531af9a48a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f33.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7a103274-02b2-436e-8e59-a0531af9a48a (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d33.sanity-pcc/d33.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.doc}&size<{1M} file: /mnt/lustre/myfile.doc, type: none file: /mnt/lustre/myfile.doc, type: none Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size<{5M}&size>{3M} Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size={5M 3M} Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d33.sanity-pcc is idle now PASS 33 (7s) == sanity-pcc test 34: Cache rule with comparator (>, <) for Project ID range ========================================================== 16:41:22 (1746132082) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions oleg353-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory oleg353-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory oleg353-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a8587000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012db75800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a8587000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012db75800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946804 s, 554 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d34.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 77170cf6-d154-49d8-abb1-9d9a37d624e2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f34.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=77170cf6-d154-49d8-abb1-9d9a37d624e2 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d34.sanity-pcc/d34.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid>{100} 99 - /mnt/lustre/f34.sanity-pcc 101 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{100} 102 - /mnt/lustre/f34.sanity-pcc 99 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{120}&projid>{110} 105 - /mnt/lustre/f34.sanity-pcc 121 - /mnt/lustre/f34.sanity-pcc 115 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d34.sanity-pcc is idle now PASS 34 (57s) == sanity-pcc test 35: mmap fault test =================== 16:42:19 (1746132139) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0978246 s, 536 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d35.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f72cf8c0-f0de-453b-a189-59430ec79dbd Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f35.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f72cf8c0-f0de-453b-a189-59430ec79dbd (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d35.sanity-pcc/d35.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d35.sanity-pcc is idle now PASS 35 (5s) == sanity-pcc test 36a: Stale RW-PCC copy should be deleted after remove the PCC backend ========================================================== 16:42:24 (1746132144) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901693 s, 581 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5c9789ce-903c-4cf3-9a32-51263a904d97 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5c9789ce-903c-4cf3-9a32-51263a904d97 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a52' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc' check file 1:'0x200004a51:0x3:0x0' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52' Handle the file: 0x200004a51:0x3:0x0 Remove non-cached file: /mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52/0x200004a51:0x3:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36a.sanity-pcc is idle now PASS 36a (6s) == sanity-pcc test 36b: Stale RO-PCC copy should be deleted after remove the PCC backend ========================================================== 16:42:30 (1746132150) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806421 s, 650 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d17ba8f0-3458-4f88-bd9a-ce0893702325 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d17ba8f0-3458-4f88-bd9a-ce0893702325 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a54' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc' check file 1:'0x200004a51:0x5:0x0' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54' Handle the file: 0x200004a51:0x5:0x0 Remove non-cached file: /mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54/0x200004a51:0x5:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36b.sanity-pcc is idle now PASS 36b (5s) == sanity-pcc test 37: Multiple readers on a shared file with PCC-RO mode ========================================================== 16:42:35 (1746132155) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942391 s, 556 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d37.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b91c8e13-dac3-442e-8924-602f11091256 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b91c8e13-dac3-442e-8924-602f11091256 (extents) (64bit) (large files) (huge files) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724238 s, 724 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d37.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7f9aae15-c43a-4628-a667-ac557244f76b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=7f9aae15-c43a-4628-a667-ac557244f76b (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d37.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} pcc: - pccpath: /mnt/pcc.d37.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} file: /mnt/lustre/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 file: /mnt/lustre2/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 /mnt/pcc.d37.sanity-pcc.2 is idle now /mnt/pcc.d37.sanity-pcc is idle now PASS 37 (9s) == sanity-pcc test 38: Verify LFS pcc state does not trigger prefetch for auto PCC-RO ========================================================== 16:42:44 (1746132164) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions oleg353-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory oleg353-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b39c2000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b3ade800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b39c2000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b3ade800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601207 s, 872 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d38.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7ea5edfa-19c4-4683-9091-0939a1662b89 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f38.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7ea5edfa-19c4-4683-9091-0939a1662b89 (extents) (64bit) (large files) (huge files) file: /mnt/lustre/d38.sanity-pcc/f38.sanity-pcc, type: none QQQQQ Cleanup PCC backend on /mnt/lustre /mnt/pcc.d38.sanity-pcc is idle now PASS 38 (73s) == sanity-pcc test 39: Test Project quota on loop PCC device ========================================================== 16:43:57 (1746132237) SKIP: sanity-pcc test_39 Not support project quota on local filesystem SKIP 39 (2s) == sanity-pcc test 40: Test async open attach in the background for PCC-RO file ========================================================== 16:43:59 (1746132239) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg353-client.virtnet /mnt/lustre (opts:) Stopping client oleg353-client.virtnet /mnt/lustre opts: Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg353-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg353-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg353-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg353-server' oleg353-server: oleg353-server.virtnet: executing load_modules_local oleg353-server: Loading modules from /home/green/git/lustre-release/lustre oleg353-server: detected 4 online CPUs by sysfs oleg353-server: MODOPTS_LIBCFS= oleg353-server: Force libcfs to create 2 CPU partitions oleg353-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg353-server: oleg353-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg353-client: oleg353-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Starting client oleg353-client.virtnet: -o user_xattr,flock 192.168.203.153@tcp:/lustre /mnt/lustre2 Started clients oleg353-client.virtnet: 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a8a06800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b394c800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a8a06800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b394c800.idle_timeout=debug disable quota as required 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.28488 s, 736 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d40.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: 82ef0fb4-0185-4629-8d77-3ddb1b986a0f Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f40.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=82ef0fb4-0185-4629-8d77-3ddb1b986a0f (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d40.sanity-pcc/d40.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.434931 s, 121 MB/s 100 - /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none ldlm.namespaces.lustre-OST0000-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800b394c800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b394c800.lru_size=clear osc.lustre-OST0000-osc-ffff8800a8a06800.stats=clear osc.lustre-OST0000-osc-ffff8800b394c800.stats=clear osc.lustre-OST0001-osc-ffff8800a8a06800.stats=clear osc.lustre-OST0001-osc-ffff8800b394c800.stats=clear fail_val=1 fail_loc=0x214 llite.lustre-ffff8800a8a06800.pcc_async_threshold=5MB llite.lustre-ffff8800b394c800.pcc_async_threshold=5MB Test open attach with pcc_async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 2.08842 s, 502 kB/s Before: 0 After: 3 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none, flags: attaching /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 8s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800a8a06800.pcc_async_threshold=1G llite.lustre-ffff8800b394c800.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800b394c800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b394c800.lru_size=clear osc.lustre-OST0000-osc-ffff8800a8a06800.stats=clear osc.lustre-OST0000-osc-ffff8800b394c800.stats=clear osc.lustre-OST0001-osc-ffff8800a8a06800.stats=clear osc.lustre-OST0001-osc-ffff8800b394c800.stats=clear Test open attach with async_threshold=1G file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: readonly, PCC_file: /d40.sanity-pcc/59f3/0x2000059f1:0x2:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00777903 s, 135 MB/s Before: 50 After: 50 Time1: 2 Time2: 10 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800a8a06800.pcc_async_threshold=5MB llite.lustre-ffff8800b394c800.pcc_async_threshold=5MB ldlm.namespaces.lustre-OST0000-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800b394c800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b394c800.lru_size=clear Read 1MB data with async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 1.05863 s, 991 kB/s /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 9s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800a8a06800.pcc_async_threshold=1G llite.lustre-ffff8800b394c800.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800b394c800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800b394c800.lru_size=clear Read 1MB data with async_threshold=1G 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00782185 s, 134 MB/s Time1: 2 Time2: 10 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d40.sanity-pcc is idle now PASS 40 (110s) == sanity-pcc test 41: Test mtime rule for PCC-RO open attach with O_RDONLY mode ========================================================== 16:45:49 (1746132349) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730594 s, 718 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d41.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 26763aa9-510a-4e07-9b67-76f877b79c84 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f41.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=26763aa9-510a-4e07-9b67-76f877b79c84 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d41.sanity-pcc/d41.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: mtime>{1m} pcc_ro_data pcc_ro_data Cleanup PCC backend on /mnt/lustre /mnt/pcc.d41.sanity-pcc is idle now PASS 41 (4s) == sanity-pcc test 42: PCC attach without attach ID specified ========================================================== 16:45:53 (1746132353) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0819182 s, 768 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d42.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: a3fc1e0d-1bcd-40c9-91da-92fd2215b9c1 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f42.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a3fc1e0d-1bcd-40c9-91da-92fd2215b9c1 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d42.sanity-pcc/d42.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} file: /mnt/lustre/f42.sanity-pcc, type: readonly, PCC_file: /d42.sanity-pcc/59f5/0x2000059f1:0x4:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d42.sanity-pcc is idle now PASS 42 (4s) == sanity-pcc test 43: Auto attach at open() should add capacity owner check ========================================================== 16:45:57 (1746132357) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0760743 s, 827 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d43.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 6bbbcf72-d067-4d01-ae0d-e3f8a7ea6710 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f43.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6bbbcf72-d067-4d01-ae0d-e3f8a7ea6710 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d43.sanity-pcc/d43.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: size<{100M} file: /mnt/lustre/f43.sanity-pcc, type: none running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f43.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f43.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/f43.sanity-pcc] file: /mnt/lustre/f43.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d43.sanity-pcc is idle now PASS 43 (4s) == sanity-pcc test 44: Verify valid auto attach without re-fetching the whole files ========================================================== 16:46:01 (1746132361) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0933421 s, 674 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d44.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 19421ccc-3e29-4bf9-bac2-4f0073cb06ee Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f44.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=19421ccc-3e29-4bf9-bac2-4f0073cb06ee (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d44.sanity-pcc/d44.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8a06800.pcc_async_threshold=0 llite.lustre-ffff8800b394c800.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443835 s, 118 MB/s /mnt/lustre/f44.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 420 0x1a4 0x280000400 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.423079 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33948 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.421539 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.341692 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.397847 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342868 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.400223 s, 131 MB/s , 0.408428 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.426052 s, 123 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409293 s, 128 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.349357 s, 150 MB/s , 0.405939 s, 129 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.353465 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410288 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.40313 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.441175 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163653 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338111 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419298 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257192 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281254 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344426 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408307 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328367 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311733 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718492 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348115 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032477 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555708 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374155 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552237 s, 949 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633536 s, 828 MB/s 52428800 bytes (52 MB) copied, 0.0341433 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0367301 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891758 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228378 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581518 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238868 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216557 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157333 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272643 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453317 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550998 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151917 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249412 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208869 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174363 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330996 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379137 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06653 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437359 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251975 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181908 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354854 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186606 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178726 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0396594 s, 1.3 GB/s , 0.0187539 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316722 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027424 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435508 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674582 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384813 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342958 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171597 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372648 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378218 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246176 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212632 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244362 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256846 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243505 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287551 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156577 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251503 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345354 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541968 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440579 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321657 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332824 s, 1.6 GB/s , 0.0340277 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278529 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363691 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399164 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038235 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0316107 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292382 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411937 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154501 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421955 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275059 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254708 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609027 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316142 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427468 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268683 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220744 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271084 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304818 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148857 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691469 s, 758 MB/s 52428800 bytes (52 MB) copied, 0.0418481 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155186 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447037 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235948 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487728 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595171 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400264 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354846 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194692 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255874 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265415 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283223 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312707 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353569 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372395 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250693 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487269 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287511 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484844 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036735 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378213 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800783 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271529 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429476 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452561 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211116 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019549 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172094 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408667 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439231 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522137 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654964 s, 800 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417118 s, 1.3 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474392 s, 1.1 GB/s , 0.0293023 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429528 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205473 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427284 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219449 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490901 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620609 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532438 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732342 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198152 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197689 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054565 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593055 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593245 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360982 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644725 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187914 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219494 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211723 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520205 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226816 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472056 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547257 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555041 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203762 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264021 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178385 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192704 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277494 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177268 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253505 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236331 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298916 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277687 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321566 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0144244 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191718 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019951 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217099 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199559 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183201 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199973 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186643 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385616 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279365 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350272 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213771 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189503 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227209 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190087 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021528 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199179 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255581 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436413 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184757 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179526 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342482 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0403251 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269091 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147717 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190363 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232823 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194412 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267289 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212942 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050568 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297371 s, 1.8 GB/s , 0.0269876 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358157 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434265 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248101 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250072 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214727 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241464 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182946 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278644 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178988 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304733 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287831 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421655 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273801 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0190456 s, 2.8 GB/s , 0.029137 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302716 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318482 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038348 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292003 s, 1.8 GB/s , 0.0312079 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350614 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356009 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531334 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211576 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024995 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273937 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201064 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321535 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237254 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319509 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260358 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186724 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240267 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375013 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199924 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285817 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018474 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206111 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278514 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017103 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187388 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291676 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235301 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028542 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240027 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277165 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.023281 s, 2.3 GB/s , 0.0234226 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438715 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214966 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168916 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278185 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232946 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171104 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183628 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165803 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424358 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231055 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185623 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298726 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269029 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233712 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223657 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454225 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027787 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353386 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031514 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465474 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438371 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540805 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251065 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234583 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474785 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492862 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308998 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220577 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245068 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206604 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187307 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261726 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568001 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203178 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334358 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293325 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405677 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218732 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242795 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228213 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193307 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255473 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296779 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167091 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567018 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107394 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0905533 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347786 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410464 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421562 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220225 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357886 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261866 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301986 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526616 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273812 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247218 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224335 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280385 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213092 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322591 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321649 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234362 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361074 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260865 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249457 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232832 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189271 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258648 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0220274 s, 2.4 GB/s , 0.0230102 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201208 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313994 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0490242 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279348 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194798 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308348 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484598 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249907 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214593 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196979 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284704 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583249 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060678 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599947 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685468 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217117 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193191 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051967 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418032 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575105 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510178 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174481 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159367 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159788 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343304 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.037258 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343161 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01793 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276137 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233106 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220264 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309545 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390283 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449499 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0209376 s, 2.5 GB/s , 0.0247218 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322549 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025977 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267223 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024245 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210115 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201354 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202912 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199177 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251521 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230388 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377121 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044184 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235367 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197606 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237538 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297079 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220026 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191226 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191583 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354722 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249061 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392922 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280401 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154229 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327758 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378512 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206202 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021678 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363139 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337017 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359043 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508504 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171123 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222483 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169561 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022528 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212564 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155547 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191099 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199012 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341682 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538355 s, 974 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245635 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393351 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.030207 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0573334 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399698 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489808 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283674 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184485 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309034 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250467 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440818 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465151 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159739 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234944 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264298 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227654 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185431 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016959 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258208 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277459 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191512 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0144649 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202009 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234335 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238608 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342323 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418235 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294541 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226098 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019907 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331625 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195991 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327408 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038645 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378158 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0672878 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162268 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172788 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355949 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332384 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200218 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209883 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295094 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017672 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353395 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365309 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245555 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253112 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208342 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0410009 s, 1.3 GB/s , 0.0355712 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581497 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690105 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073999 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233214 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324229 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342785 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310819 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389598 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209338 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021725 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025173 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505634 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510247 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304585 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276027 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395938 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270872 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023857 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279199 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235676 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202368 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193929 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270811 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028242 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235349 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319979 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462165 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483448 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451403 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575449 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423645 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210884 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742565 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643393 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326151 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893549 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289059 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453704 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542806 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370257 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388347 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391224 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287067 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277289 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025805 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369997 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300793 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243887 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357262 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242241 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275687 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268685 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333066 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233709 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195685 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214229 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216041 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194294 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291814 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469004 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595721 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330429 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243891 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033875 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0957612 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409786 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485318 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816488 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265118 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623535 s, 841 MB/s 52428800 bytes (52 MB) copied, 0.0542123 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335687 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250136 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262061 s, 2.0 GB/s , 0.0249341 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218261 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244398 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305771 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271169 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282346 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209537 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01931 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206796 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182725 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338749 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320801 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026708 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191078 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187195 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023961 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388104 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158387 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179177 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0183049 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0659 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158658 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356711 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444969 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540333 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285508 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321307 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304685 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154886 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393543 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410701 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519374 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229338 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241397 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034361 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203712 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291887 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310983 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369374 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432619 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040455 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606571 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378909 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585221 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271547 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240109 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217044 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163041 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221383 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179399 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196209 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201676 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281492 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229952 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277561 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161557 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371761 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452216 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310515 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308316 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219746 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182594 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166638 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264334 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257171 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247485 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172372 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248759 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308686 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205004 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181331 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238357 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026245 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270528 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203309 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023201 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479687 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190254 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223348 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376535 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237921 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275636 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364312 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453318 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601661 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206073 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311679 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401299 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528447 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258598 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344647 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048467 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554468 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612654 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299399 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186616 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621081 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263916 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410446 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683248 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531304 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876262 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678165 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211358 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308992 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025574 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295279 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444487 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230535 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542953 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272643 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419082 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567354 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365453 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600397 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274222 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239534 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024618 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385892 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190805 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370939 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452495 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509849 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241785 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225387 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223592 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228716 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273406 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187813 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017437 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272339 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282719 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0159136 s, 3.3 GB/s , 0.0320332 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273786 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275475 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169094 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342226 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282557 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178628 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271306 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256765 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282011 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478746 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233721 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372851 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561343 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757364 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291666 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389745 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519497 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287876 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367675 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0310708 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312607 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258839 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018043 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188613 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194589 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380628 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417957 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341165 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0645726 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287985 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288526 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0271093 s, 1.9 GB/s , 0.0183945 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314891 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271372 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287038 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286403 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574434 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426411 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388739 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374848 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307526 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208372 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531553 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171262 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219228 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170228 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0552981 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228077 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351279 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288741 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161893 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027618 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027014 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166876 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523339 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372619 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676867 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280344 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292707 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429199 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522852 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206999 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270888 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167754 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392809 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265555 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020576 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247822 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257139 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309311 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391444 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548669 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189333 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371543 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307197 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373355 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198375 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05369 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192932 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424532 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394914 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395767 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580763 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251215 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0879761 s, 596 MB/s , 0.0405817 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442261 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191491 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236833 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686616 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309164 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0220761 s, 2.4 GB/s , 0.0339636 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357715 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704906 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510458 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310566 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218406 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284541 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464271 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396102 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430444 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633786 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535095 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06128 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249615 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221747 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517061 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560488 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385389 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428775 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050817 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209167 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224735 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226751 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0375492 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202421 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316181 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486014 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303878 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332743 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256285 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323094 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227496 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403615 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188897 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314957 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239164 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657489 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019258 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257549 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492751 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180585 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534437 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207804 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304753 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313036 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411563 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320006 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188454 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343804 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443519 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197029 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336667 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043906 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476088 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258709 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526464 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265809 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166733 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0339615 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04171 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280647 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539532 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198138 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506839 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312312 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197685 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267041 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187236 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275807 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265837 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382904 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299026 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518388 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190588 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0638727 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235097 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349134 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223731 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296677 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235702 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243646 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207111 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210475 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425995 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03177 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371272 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280199 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282436 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480709 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261183 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376415 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304522 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252442 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253966 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204241 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171074 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262621 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01923 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248212 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230292 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240233 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234792 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264013 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216446 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263512 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267395 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296403 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236831 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268577 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291426 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263161 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304604 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328101 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157957 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302339 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033072 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265693 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271996 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211953 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186423 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289554 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368623 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199983 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302138 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016761 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226617 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221285 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361502 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202921 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223203 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223927 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291595 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451113 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484148 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206178 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298789 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234946 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034183 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325228 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205943 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242967 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426465 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033794 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382239 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027581 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274102 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235179 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287307 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185811 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193066 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021466 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248655 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236818 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168955 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0192848 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.018526 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208121 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256152 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322963 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242486 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814517 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347322 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407772 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039898 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193518 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243878 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222387 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313968 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216294 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223213 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042376 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498376 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532785 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033672 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227054 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365553 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640833 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186882 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341898 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434185 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413112 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239016 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318208 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219113 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434677 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495437 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185967 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417252 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421191 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375115 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033035 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274677 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310157 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233146 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392883 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032595 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308184 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247726 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243924 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261668 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306411 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023095 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218482 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182525 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293678 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232469 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412458 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368416 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042153 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715554 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207995 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483969 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029728 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494113 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308224 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284766 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461589 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374646 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335505 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195219 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322472 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280663 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459294 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273308 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326642 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253403 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266411 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168515 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207833 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163606 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156999 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201269 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191212 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266351 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334024 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211052 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212941 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211195 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264899 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218634 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304537 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271529 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204389 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527837 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169453 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380276 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0276758 s, 1.9 GB/s , 0.0418656 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204409 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249206 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210358 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306752 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209256 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324438 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464815 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261773 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149993 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025689 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154279 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328788 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238245 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559623 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453991 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307726 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388058 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301069 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387188 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237142 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460979 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051427 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248008 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368427 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230595 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581796 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718013 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311538 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367544 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384091 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025914 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474683 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360826 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375602 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408093 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440478 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530681 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828242 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259542 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321317 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576504 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319921 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350794 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359803 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387723 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365064 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269633 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208586 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289331 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437166 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446275 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411234 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404308 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269676 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393666 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385898 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308207 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701566 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592685 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356416 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404236 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377671 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398109 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447313 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469263 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367462 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347967 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414754 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537967 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567032 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249128 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347575 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466399 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505734 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037442 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431598 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227522 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203746 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199547 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219229 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414885 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246755 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308844 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276028 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031692 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291315 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434757 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302899 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271968 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253971 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182865 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185864 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306932 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320181 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344621 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346426 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304256 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320265 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354213 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222585 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361521 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155258 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171126 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237492 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210013 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223677 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235072 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386724 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422039 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226916 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030435 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175538 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265612 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218656 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284021 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429485 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529449 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335427 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377092 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340643 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189049 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207717 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285687 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809907 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201583 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0295003 s, 1.8 GB/s , 0.0390352 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299628 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231182 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223681 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184539 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174113 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188495 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207638 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247683 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450708 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449113 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637357 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444471 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375923 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356302 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269164 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228481 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364099 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281119 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276355 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053543 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421243 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485034 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196106 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0227353 s, 2.3 GB/s , 0.0366198 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250646 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249554 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448029 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595217 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109602 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114305 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894517 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0915543 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0972299 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103885 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629623 s, 833 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385008 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.065203 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264791 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344612 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02577 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198369 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335012 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349835 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028988 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350987 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298746 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218524 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223809 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207521 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199469 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166614 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156783 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259493 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307508 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177937 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346053 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021161 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293166 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287682 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402021 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303035 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190026 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205211 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210594 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302465 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269876 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356909 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026237 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482571 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348241 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022506 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281237 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02382 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207854 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185747 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200858 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206457 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209998 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228711 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169795 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293257 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242611 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238921 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231642 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223915 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289131 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178973 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170421 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232105 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170336 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217323 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186039 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183822 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174992 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019269 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027681 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215785 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213897 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147294 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168597 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187434 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168342 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027887 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240228 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238056 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222793 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173839 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259277 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218465 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274954 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236158 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221011 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160143 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240785 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187937 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207412 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163984 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201525 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023127 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021633 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272542 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310104 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220066 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275943 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02702 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.02642 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035376 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365523 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201253 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264801 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319979 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017104 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227816 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251196 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201401 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018708 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258259 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274321 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285372 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262031 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368538 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420966 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400494 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433901 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354328 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196811 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298155 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022595 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205079 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251024 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358698 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241076 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303729 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325104 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172393 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215681 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202106 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018896 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314464 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190883 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023596 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513461 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220887 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279605 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.028945 s, 1.8 GB/s , 0.024176 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188649 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175041 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211482 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214245 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246812 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209732 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219871 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202151 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266576 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213912 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237569 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204821 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022384 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205309 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250398 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0316326 s, 1.7 GB/s , 0.0348002 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337853 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168123 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294936 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199437 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300298 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328088 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214402 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238328 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.027085 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270057 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226409 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268074 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197645 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274435 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350948 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318001 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365726 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038422 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218386 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328609 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019961 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230526 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044967 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673953 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275505 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221527 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198523 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.017517 s, 3.0 GB/s , 0.0170167 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025061 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309885 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387492 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527128 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282088 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537785 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351474 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253549 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259656 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419357 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382114 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398198 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301589 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045986 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276382 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026937 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252074 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361242 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321295 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238482 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300419 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358025 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283342 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394482 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437897 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280488 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044861 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0309421 s, 1.7 GB/s , 0.0480816 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380691 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510267 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378861 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212133 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032243 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325357 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387549 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179213 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210679 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277094 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158666 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297249 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354356 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201111 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299708 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282468 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334474 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397565 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372606 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549053 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591581 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763236 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196792 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215636 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025486 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282366 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281259 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.02079 s, 2.5 GB/s , 0.0234518 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271633 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279694 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169131 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248997 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306758 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217805 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214553 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220505 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198584 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202772 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270887 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190863 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188801 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198699 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300163 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213117 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241203 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024492 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172947 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0229967 s, 2.3 GB/s , 0.0211407 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0227097 s, 2.3 GB/s , 0.0258047 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262825 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178921 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213597 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413464 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383527 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230292 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284999 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437955 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267608 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205504 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206942 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655376 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260052 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017573 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379486 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261861 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192357 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348272 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229432 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472148 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336593 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040759 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252399 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171062 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390637 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191607 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506084 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245626 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233571 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209952 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229243 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398419 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031846 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300385 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238325 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337023 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355156 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322983 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330478 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240236 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219837 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286713 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488496 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454386 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271825 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381001 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366193 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315814 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347502 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0278391 s, 1.9 GB/s , 0.0357321 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417151 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334799 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404266 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216663 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283102 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048689 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437123 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322611 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430341 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0506186 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245885 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01935 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392052 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397329 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479272 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284737 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243779 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220758 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217823 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261962 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348258 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341731 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239989 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221368 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178724 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169035 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177219 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164708 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180927 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202989 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224776 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235018 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334907 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303185 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265114 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221323 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270378 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283068 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310009 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355178 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172198 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205535 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242799 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313306 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274305 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297529 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166267 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161825 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270708 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228854 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027568 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020768 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256275 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303743 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199175 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236928 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190555 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317091 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222216 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180931 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276289 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185709 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226016 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308194 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174843 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0155712 s, 3.4 GB/s 52428800 bytes (52 MB) copied, 0.0210535 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232279 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022789 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220376 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019208 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214375 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238195 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213851 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155786 s, 3.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039194 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425589 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0455125 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016931 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296713 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309658 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266716 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170788 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200002 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219654 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204369 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262633 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419358 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177072 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024072 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240413 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216189 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219629 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204077 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202068 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276804 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358031 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259831 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246669 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305252 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298214 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216348 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173805 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024355 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226105 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267479 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220487 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369534 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344308 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301678 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224205 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235129 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221428 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259559 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211661 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170577 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241367 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028538 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216095 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024958 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260938 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319805 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030333 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173351 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174699 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242973 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256688 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263288 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192931 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182676 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236195 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174298 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183383 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234695 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343262 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196986 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0188954 s, 2.8 GB/s , 0.0187526 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203906 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204736 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022699 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212689 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353048 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054584 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234548 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258345 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170427 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193775 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267351 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250448 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317569 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212605 s, 2.5 GB/s , 0.0179762 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165005 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172584 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358539 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376212 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196107 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222242 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226989 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357484 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224394 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200265 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308342 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363728 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259093 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193777 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257373 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306667 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373203 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237801 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307979 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265309 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192652 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349474 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373108 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611518 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174158 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231707 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027985 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361963 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266285 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01921 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173243 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029755 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307952 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260489 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179748 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217429 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321281 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253742 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204107 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232806 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179109 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239506 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236162 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245476 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293041 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177566 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215223 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372014 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173766 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304052 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227623 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247095 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289529 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215554 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280982 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295437 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02975 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322826 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228775 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026056 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374958 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204455 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160524 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207382 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020655 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222038 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039383 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206452 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289666 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181064 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366692 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496612 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020741 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313884 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345652 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414324 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386982 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225973 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272319 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207003 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222602 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229867 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280852 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213269 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340262 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214788 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342299 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293102 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302674 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271704 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203152 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236234 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367164 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298873 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253736 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343957 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463707 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535663 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019052 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334419 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195366 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200293 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250534 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205053 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253256 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462614 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380956 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221526 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335495 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299322 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427409 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047539 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397437 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293868 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464296 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369237 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398403 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209573 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354553 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355932 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322047 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198378 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336402 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312897 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215773 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223723 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282374 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174465 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262141 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266687 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368074 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333201 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202517 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220452 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205898 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410223 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314718 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018362 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023952 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321464 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242288 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287622 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274202 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242647 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403758 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0500834 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308249 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299051 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360592 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020245 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191119 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207995 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0219991 s, 2.4 GB/s , 0.0215291 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217266 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321981 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225867 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282218 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398057 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296824 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624308 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603447 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299181 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432816 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446085 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278204 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402032 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182765 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255621 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192762 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305847 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465564 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410341 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611808 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378886 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607008 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378905 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303753 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337582 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364346 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031135 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220633 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298192 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279907 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0399964 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0460779 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277169 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384289 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465407 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534658 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467692 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528819 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582361 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382291 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214521 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435306 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211683 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0309932 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350373 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0312837 s, 1.7 GB/s , 0.0353434 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265498 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273315 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218641 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264372 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296507 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187202 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286923 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409977 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284203 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030162 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158345 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026125 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032641 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389158 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426959 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599174 s, 875 MB/s 52428800 bytes (52 MB) copied, 0.0582354 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216223 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029567 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449498 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409261 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244788 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323343 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324115 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369188 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249155 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301139 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278275 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235772 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373391 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367574 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383145 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364051 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017038 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250874 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274905 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435109 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327511 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300096 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345432 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322155 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208909 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569966 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558307 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206021 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441596 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363942 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184079 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238393 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184796 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029301 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309105 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280257 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232269 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213275 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315433 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290099 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376556 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263644 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252842 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360764 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238598 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234354 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304354 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257844 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193621 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208333 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378414 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047695 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260185 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219278 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296993 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154943 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249385 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194467 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264137 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203611 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261016 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363887 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502223 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443826 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242149 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447143 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199078 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184197 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232982 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206963 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204006 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320529 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234348 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347598 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228808 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364644 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216069 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232629 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364094 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322922 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171982 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017877 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229101 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229767 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206914 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023629 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294485 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614338 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345806 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436661 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343552 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376772 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198461 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210185 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343637 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188397 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186129 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314096 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258465 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285664 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332491 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209953 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328509 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400676 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315395 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253462 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241665 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198522 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259117 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247157 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022915 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417136 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307249 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514669 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543518 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343721 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247117 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217722 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271582 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381464 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252929 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301913 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337429 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585244 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545667 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658244 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537976 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253235 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457783 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392355 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264742 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042776 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645156 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336885 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419308 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031946 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236963 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252008 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258767 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556501 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380951 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192072 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428994 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296195 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309075 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027975 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171223 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267081 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020214 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226079 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228494 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029041 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329791 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343772 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345371 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253116 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172145 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314506 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297431 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355817 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334893 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321928 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033904 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364087 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272016 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498033 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243355 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363356 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360446 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270044 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443611 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326099 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452447 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200575 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040279 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412569 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335853 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491531 s, 1.1 GB/s , 0.0207363 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420765 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172987 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222681 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347454 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330861 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277987 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303605 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196165 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278435 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256454 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022031 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198233 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296569 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192329 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202694 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220701 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216919 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173229 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345444 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212044 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224409 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217909 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190305 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161262 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355962 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035443 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635815 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078887 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032612 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815253 s, 643 MB/s 52428800 bytes (52 MB) copied, 0.0880237 s, 596 MB/s 52428800 bytes (52 MB) copied, 0.0326634 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295912 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234545 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416387 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339862 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528945 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212729 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285989 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443311 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283887 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402381 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340366 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345326 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517149 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183254 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376685 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221897 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253681 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387164 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316231 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021495 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344284 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223712 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153071 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274246 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188507 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241227 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176765 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186253 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036514 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201744 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493299 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458438 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212198 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254502 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190361 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184989 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169486 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167722 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232756 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195282 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282096 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211102 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436029 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0219481 s, 2.4 GB/s , 0.0219197 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208364 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435997 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289333 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249492 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220301 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329635 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247599 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018916 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315425 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226506 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302893 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302103 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340539 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279693 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635475 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314714 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440495 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344531 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299115 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552622 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501998 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328747 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381827 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494299 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368185 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394638 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203319 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291121 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208604 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214763 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413189 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329901 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157669 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021173 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175618 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0167311 s, 3.1 GB/s , 0.0193304 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170256 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276221 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419315 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040819 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499272 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444173 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310764 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557142 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479374 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198477 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782136 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274142 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269323 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459858 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222579 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049409 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187284 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207414 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273365 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233153 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215169 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364067 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0414371 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284112 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322953 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0193162 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293513 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018523 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217719 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217069 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235916 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289781 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344645 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315088 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362575 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373801 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160557 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483095 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277299 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022704 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269819 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212796 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164774 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402692 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184922 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229315 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339882 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252934 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188819 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202265 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501183 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290557 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267888 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213965 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318684 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482811 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0276212 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015774 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391431 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201638 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231107 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224809 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202037 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226923 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264711 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213195 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678133 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706745 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363583 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370045 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440264 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432573 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457396 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0374629 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247049 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244038 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204552 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186473 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220096 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314549 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338286 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297566 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304908 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360883 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404507 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376164 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182873 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422506 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358114 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314564 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215633 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286547 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393896 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234401 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547487 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233225 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272233 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316599 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190119 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320184 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300114 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355478 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408504 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491504 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737508 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544506 s, 963 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0269851 s, 1.9 GB/s , 0.0245987 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460132 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484661 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241596 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360702 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430166 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493015 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408523 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171771 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196637 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181192 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060439 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572312 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162461 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404714 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293857 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213479 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0239007 s, 2.2 GB/s , 0.0233424 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247677 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166045 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019715 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193055 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229113 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381775 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256422 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306196 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325163 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295962 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232773 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167278 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250849 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158846 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248924 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240213 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227501 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185834 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328882 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340061 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356359 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214764 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241257 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475172 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421487 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618719 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297221 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433929 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555007 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448508 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158573 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286619 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01683 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022233 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033253 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207324 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154911 s, 3.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301211 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0273903 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0244768 s, 2.1 GB/s , 0.0303657 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222725 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186382 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239017 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034392 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249174 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356459 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372273 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343275 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332903 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322866 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232013 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227761 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432906 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035916 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504086 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278191 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208054 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304596 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244693 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334767 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399813 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440654 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454555 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503237 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162062 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486491 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562237 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394308 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572229 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744098 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653254 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689136 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708978 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490595 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03243 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187573 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335102 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389872 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588012 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734218 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208592 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0790988 s, 663 MB/s , 0.0712362 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423993 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333391 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557324 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368568 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309711 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231616 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292914 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342659 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537113 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437329 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491497 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265755 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794792 s, 660 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15472 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124399 s, 421 MB/s 52428800 bytes (52 MB) copied, 0.127134 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169279 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326878 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042307 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632602 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.152675 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107904 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294768 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553397 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613926 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886404 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592906 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947164 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101242 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313481 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409748 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041016 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321607 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362812 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699207 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663806 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201915 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389181 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234258 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282833 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0932433 s, 562 MB/s , 0.0323501 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450379 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438341 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502959 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220081 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191857 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347172 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571256 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872905 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506421 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030299 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272436 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362775 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174566 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379239 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433671 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208037 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240485 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226597 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249988 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242586 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342086 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219257 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649105 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631551 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166646 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266234 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486573 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332936 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377164 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360788 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226005 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193903 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264167 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192602 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457509 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376588 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434638 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177915 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313485 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268076 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267765 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177169 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227475 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285377 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177484 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353381 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038154 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406873 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170059 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194099 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206019 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034983 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479862 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191951 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248218 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329471 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331772 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161174 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274856 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348185 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279756 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419507 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203332 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299888 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314685 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263278 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351244 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555465 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334926 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461715 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440781 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161732 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349775 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403039 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034166 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267193 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476955 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215374 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211327 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339814 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315298 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327464 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218147 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278041 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035627 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277566 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04162 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236433 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234688 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279335 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180019 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031809 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545582 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623987 s, 840 MB/s 52428800 bytes (52 MB) copied, 0.0270968 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425604 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520633 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226131 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155237 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213402 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175074 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310959 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155873 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279832 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365679 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321216 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238144 s, 2.2 GB/s , 0.0317836 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168675 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344861 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465369 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332372 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626266 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336226 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298648 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495759 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543252 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173559 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405818 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400571 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225263 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217508 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161618 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054253 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468364 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360312 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015375 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542975 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578857 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644719 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372252 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403999 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879863 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561577 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307061 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044327 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480898 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392454 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066465 s, 789 MB/s 52428800 bytes (52 MB) copied, 0.0321035 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892032 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278619 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704162 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818875 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625888 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574091 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750591 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168714 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555621 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328201 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666293 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394038 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417345 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091675 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102669 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122996 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398075 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075156 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253601 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301055 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922882 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778321 s, 674 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111863 s, 469 MB/s 52428800 bytes (52 MB) copied, 0.0719935 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210608 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608117 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536352 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449911 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213781 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376083 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374334 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272047 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285243 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186132 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465365 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789751 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103082 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120947 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155463 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603956 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130179 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131684 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569982 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610223 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126764 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456653 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359542 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429339 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420065 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507402 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378577 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610121 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485848 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055647 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489877 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208777 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213945 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932634 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025325 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180016 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406101 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410319 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875345 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361956 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745867 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441096 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09705 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12873 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217174 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423566 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434948 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194521 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325171 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271591 s, 1.9 GB/s , 0.0309654 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376322 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307326 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175163 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271476 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019126 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185289 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187291 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199965 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192474 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231038 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234106 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385581 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232674 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189803 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369393 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320421 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240814 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179965 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705533 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277826 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428211 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0563211 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345629 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441477 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025669 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261204 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223364 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331011 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270751 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389819 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160803 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233288 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228944 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266957 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193688 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198335 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493181 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281466 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419934 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050903 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284548 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287743 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338086 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290294 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325389 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237171 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353689 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295377 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0271463 s, 1.9 GB/s , 0.0263283 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159814 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155769 s, 3.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0325465 s, 1.6 GB/s , 0.024528 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194559 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021146 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244139 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529607 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368107 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367009 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454918 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219928 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380262 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353828 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291282 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287974 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184361 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400254 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023682 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262266 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259383 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209199 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0149239 s, 3.5 GB/s , 0.0304391 s, 1.7 GB/s , 0.019159 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373746 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192117 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300383 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280411 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319906 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037769 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311291 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247713 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332011 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272471 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023552 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375098 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254822 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259874 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298202 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169235 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241847 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151563 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174978 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352398 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161474 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197954 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360341 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179618 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299141 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181152 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262046 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154624 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216196 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230533 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255694 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030369 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245351 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302319 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250194 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189209 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252874 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201475 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206287 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453233 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380309 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177836 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201232 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213947 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018807 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262972 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263654 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328734 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258106 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248784 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340091 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616958 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376887 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495111 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178561 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0380134 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228014 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416993 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535099 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440624 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291879 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512723 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396449 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622785 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646027 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407301 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160944 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504454 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234053 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035763 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253184 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287649 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460346 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345462 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464931 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533568 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247399 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466501 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049328 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246144 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612299 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058937 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562695 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448654 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103576 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0862653 s, 608 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882434 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182436 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0802921 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855296 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125065 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088068 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184199 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327589 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197956 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556882 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416563 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025275 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445014 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035488 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250811 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188686 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047074 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222273 s, 2.4 GB/s , 0.0204728 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363327 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177669 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219628 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209354 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404312 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031393 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194445 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360526 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448843 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228378 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277394 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289296 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308076 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242446 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275942 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454854 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182485 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191744 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163002 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025648 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164031 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405814 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235652 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286866 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222252 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709924 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611283 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0790725 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563767 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667585 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486711 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337676 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529368 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442559 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483913 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434723 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374004 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675796 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.014706 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219494 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198327 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171648 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270071 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319323 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188523 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165888 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0281682 s, 1.9 GB/s , 0.0262379 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218959 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271963 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020086 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261541 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148408 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.014797 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235944 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226744 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304649 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177454 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189945 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173521 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218844 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203959 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032838 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169307 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210873 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158681 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161916 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019266 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152049 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218048 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342825 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292505 s, 1.8 GB/s , 0.0212157 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168585 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178438 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160735 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181366 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246201 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280793 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213788 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020885 s, 2.5 GB/s , 0.0308511 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233859 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016608 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02103 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239018 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281746 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021591 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284684 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242104 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352106 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193042 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172023 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249489 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284039 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175576 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218636 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186788 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256679 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01638 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184854 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228299 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0353996 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156773 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192121 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253881 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169336 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205261 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198648 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019852 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220837 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166069 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0322065 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0174635 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182507 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184818 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178808 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158484 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163993 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165876 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154577 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184221 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173856 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158767 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189281 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167127 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328549 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0150992 s, 3.5 GB/s , 0.0154873 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021146 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161665 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356117 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193044 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220038 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180059 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171196 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218421 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177489 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223647 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279118 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277067 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179989 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202187 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168548 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210654 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019684 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215682 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171535 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174928 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267472 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272135 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194131 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018023 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178782 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216171 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0198443 s, 2.6 GB/s , 0.0160853 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172682 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191761 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175546 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194393 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194501 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020375 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245507 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392271 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396532 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0530795 s, 988 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029352 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228631 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0221063 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460452 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322426 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177337 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243059 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184057 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023694 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198669 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260397 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374612 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387294 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0390695 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0253609 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0316465 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0470754 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332768 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018683 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185748 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183879 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327751 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337696 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193626 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018833 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156287 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196711 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201649 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160276 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152011 s, 3.4 GB/s 52428800 bytes (52 MB) copied, 0.0159404 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161392 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188113 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160935 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286376 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161795 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185081 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193602 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017753 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150955 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218953 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167778 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177538 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215704 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215273 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218181 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250064 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346178 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022642 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299397 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171255 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201288 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163859 s, 3.2 GB/s , 0.0158221 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180702 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166241 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198265 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227426 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0178871 s, 2.9 GB/s , 0.0199115 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187609 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172235 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160203 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197651 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151669 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016532 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168969 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187367 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199035 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179762 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328212 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277003 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292604 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166874 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168644 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186077 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400764 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390685 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301745 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329181 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295473 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025176 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296926 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252628 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178545 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228559 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202215 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035015 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237369 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227646 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237368 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187341 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305031 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173148 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169581 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228658 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200471 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239108 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277534 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0149999 s, 3.5 GB/s , 0.0160916 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187687 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155256 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038657 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482972 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412387 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025449 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225412 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154184 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268264 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167021 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179276 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275796 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217119 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199043 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354933 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252786 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030228 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198486 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243501 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220365 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196963 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164573 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155656 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243004 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217733 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221164 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253177 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171036 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160007 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199857 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176572 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233485 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169975 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151306 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025556 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208359 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018891 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152436 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157597 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165922 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203872 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190444 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154155 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179762 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207513 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201538 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209163 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191043 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169782 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157081 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168458 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172568 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217322 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232133 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240032 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025199 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233361 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299504 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260543 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188598 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015864 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317989 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260491 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228621 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191091 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149592 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185364 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295713 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229538 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275542 s, 1.9 GB/s Finish ======== /mnt/lustre/f44.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x1a4:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800a8a06800.stats= snapshot_time 1746132423.903301739 secs.nsecs start_time 1746132363.395001325 secs.nsecs elapsed_time 60.508300414 secs.nsecs read_bytes 1690 samples [bytes] 1048576 1048576 1772093440 1858174650941440 read 156450 samples [usecs] 128 316742 98871949 1134394096687 ioctl 1 samples [reqs] open 3133 samples [usecs] 4 79259 4421297 83013804757 close 3133 samples [usecs] 7 39139 2598669 26231006597 seek 3128 samples [usecs] 2 27569 52948 815485010 setattr 1 samples [usecs] 11361 11361 11361 129072321 getattr 17 samples [usecs] 93 6126 14247 69093411 mknod 1 samples [usecs] 5714 5714 5714 32649796 inode_permission 9671 samples [usecs] 1 30603 3367843 24651080643 opencount 3134 samples [reqs] 1 3133 4909412 10255759580 openclosetime 3132 samples [usecs] 0 150789 65595925 2842598426833 pcc_attach 1 samples [reqs] pcc_detach 262 samples [reqs] pcc_auto_attach 262 samples [reqs] pcc_hit_bytes 3097 samples [bytes] 52428800 52428800 162371993600 8512968778055680000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff8800b394c800.stats= snapshot_time 1746132423.903410880 secs.nsecs start_time 1746132363.395021564 secs.nsecs elapsed_time 60.508389316 secs.nsecs attach 1 detach 262 auto_attach 262 llite.lustre-ffff8800a8a06800.pcc_async_threshold=1073741824 llite.lustre-ffff8800b394c800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d44.sanity-pcc is idle now PASS 44 (65s) == sanity-pcc test 45: Concurrent read access from two mount points ========================================================== 16:47:06 (1746132426) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.138356 s, 758 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d45.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: 253587a0-9d40-40a5-ba3e-2bf5394fc50d Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=253587a0-9d40-40a5-ba3e-2bf5394fc50d (extents) (64bit) (large files) (huge files) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.150013 s, 699 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d45.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: bad966a9-38b2-468b-b440-3ada1b3da7e8 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=bad966a9-38b2-468b-b440-3ada1b3da7e8 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d45.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} pcc: - pccpath: /mnt/pcc.d45.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8a06800.pcc_async_threshold=0 llite.lustre-ffff8800b394c800.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.402022 s, 130 MB/s /mnt/lustre/f45.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 421 0x1a5 0x280000400 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.244405 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250197 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247375 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.238985 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.416847 s, 126 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.467876 s, 112 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.441249 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477029 s, 110 MB/s 52428800 bytes (52 MB) copied, 0.479808 s, 109 MB/s 52428800 bytes (52 MB) copied, 0.483219 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.486371 s, 108 MB/s 52428800 bytes (52 MB) copied, 0.484701 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.447371 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.503139 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.469988 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.514168 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767594 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03222 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100918 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.579231 s, 90.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.59369 s, 88.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.592979 s, 88.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.599819 s, 87.4 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.604849 s, 86.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.605501 s, 86.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.607349 s, 86.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.608066 s, 86.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.608741 s, 86.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.608313 s, 86.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.61106 s, 85.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.612589 s, 85.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.615243 s, 85.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.618824 s, 84.7 MB/s 52428800 bytes (52 MB) copied, 0.629434 s, 83.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.622982 s, 84.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210998 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0396591 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0291865 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159936 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324141 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568279 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567318 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724576 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131539 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133421 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0998178 s, 525 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104707 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0955055 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119522 s, 439 MB/s 52428800 bytes (52 MB) copied, 0.14025 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218501 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545735 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128346 s, 408 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513308 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.129225 s, 406 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117128 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469407 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0424371 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129446 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111872 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261144 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128837 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348227 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483399 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174119 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169192 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257365 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415402 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249448 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343387 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631215 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428766 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0995473 s, 527 MB/s 52428800 bytes (52 MB) copied, 0.042265 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0548356 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667857 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434161 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0355842 s, 1.5 GB/s , 0.030197 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310776 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210402 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206546 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157638 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198942 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219314 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168447 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180285 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204036 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178414 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172641 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02439 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219244 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210695 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160399 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236259 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200995 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036514 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165726 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337033 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022381 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193926 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0376172 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0249917 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312416 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0280091 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292436 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264694 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355203 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024253 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202254 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162173 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172426 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191139 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219273 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154096 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016462 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212099 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175192 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221851 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212249 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0212128 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179233 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230023 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171384 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185802 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153312 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247851 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182013 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015304 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185742 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036227 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185557 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0144785 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016058 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222361 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241947 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0327869 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304504 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306497 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348272 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0362812 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299964 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413808 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228099 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306158 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426166 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466722 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182881 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0291164 s, 1.8 GB/s , 0.0353535 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329729 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222754 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0146231 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156084 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154287 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161473 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195859 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01556 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162094 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164814 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187359 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173717 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170837 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150486 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193099 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212978 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178258 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176672 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159394 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264297 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161715 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202208 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185296 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209167 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200994 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197376 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0152938 s, 3.4 GB/s 52428800 bytes (52 MB) copied, 0.015699 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171362 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155109 s, 3.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173016 s, 3.0 GB/s 52428800 bytes (52 MB) copied, 0.0181917 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179262 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205081 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172357 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032333 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031056 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0402825 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176705 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309304 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159188 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148257 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337873 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193385 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244269 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311211 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170343 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290958 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200168 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175023 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251212 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193149 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038776 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0298005 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231702 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499003 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062634 s, 837 MB/s 52428800 bytes (52 MB) copied, 0.0551294 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256925 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482981 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278644 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257916 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506192 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409712 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0396884 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443384 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.034654 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245491 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038358 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182229 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324821 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252517 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313719 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211553 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016545 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160941 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227095 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229396 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019725 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286959 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322064 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256452 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239496 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234676 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179351 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325452 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342107 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388171 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166937 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169803 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220166 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149836 s, 3.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228106 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386802 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.042837 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166922 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235233 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196366 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507351 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207976 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500613 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0280095 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180439 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498132 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285054 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170714 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226477 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446053 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301248 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159952 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169286 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233222 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295631 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233322 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185451 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175425 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196364 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291416 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296071 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019691 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430565 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416339 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285829 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390246 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305396 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042355 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408061 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029323 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242126 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216786 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228607 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0154097 s, 3.4 GB/s , 0.0147429 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029235 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0302922 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0315621 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434014 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020969 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268321 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409551 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359607 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240655 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346053 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397789 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433053 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302425 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224348 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217165 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258975 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169429 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153978 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202485 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159222 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179767 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348623 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017498 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275981 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181216 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183133 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019061 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183065 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246414 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383144 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252567 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417812 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030473 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241707 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461665 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049275 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426147 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0315782 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400036 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560608 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189407 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272325 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729669 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441105 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453098 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167947 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154967 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065041 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359459 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200789 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164872 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209032 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154601 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151952 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166327 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180474 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274189 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198329 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213936 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248877 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020118 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281856 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226375 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210595 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393553 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028636 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198083 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357032 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235956 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202554 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317949 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197172 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281541 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179934 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270814 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191025 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162974 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304589 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239781 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177968 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207571 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175367 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263142 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193358 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160142 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160689 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165552 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226264 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195161 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209265 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507332 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362535 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631505 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324556 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597732 s, 877 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549991 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376783 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0633479 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342794 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422702 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269032 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411656 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.020377 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179483 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439287 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201268 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251042 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149137 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280212 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210228 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178067 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0295026 s, 1.8 GB/s , 0.0206194 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385371 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354661 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197077 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299042 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218123 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152072 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173227 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155363 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168212 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227159 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184793 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201384 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196261 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033174 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294719 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497041 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505793 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572998 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417282 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756094 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545264 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576758 s, 909 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040799 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455015 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0385124 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035624 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679973 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723228 s, 725 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260572 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340342 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0285782 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486858 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401887 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531426 s, 987 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289108 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0185184 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327039 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414574 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182791 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204146 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187515 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175636 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171703 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212777 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306565 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216298 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0212853 s, 2.5 GB/s , 0.0344966 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323457 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237539 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159089 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387138 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349666 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364843 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308791 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496596 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569258 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700943 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827256 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050048 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623562 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392121 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490313 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432196 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610745 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074799 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730536 s, 718 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216798 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0318528 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.033439 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200845 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497545 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449806 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492341 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022216 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449355 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236758 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202081 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244962 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252199 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299777 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150445 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189292 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160897 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186289 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166641 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0231264 s, 2.3 GB/s , 0.0159523 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333046 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283073 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276313 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160574 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164768 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346382 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0397843 s, 1.3 GB/s , 0.0225237 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280352 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275659 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227604 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339486 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167359 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0259885 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191228 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151015 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306383 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308307 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254565 s, 2.1 GB/s , 0.0397502 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299655 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017934 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298248 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270975 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288714 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627795 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306712 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687616 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056601 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561367 s, 934 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671208 s, 781 MB/s 52428800 bytes (52 MB) copied, 0.0344496 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264177 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430688 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362978 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408841 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057133 s, 918 MB/s 52428800 bytes (52 MB) copied, 0.0419962 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181716 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0142276 s, 3.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263158 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258467 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236681 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162784 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037474 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148979 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247337 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166971 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285191 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290506 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198646 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254303 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173113 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034023 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355828 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379332 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301259 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197944 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036157 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500708 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609944 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243996 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602711 s, 870 MB/s 52428800 bytes (52 MB) copied, 0.0698442 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530172 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327019 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267028 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301446 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020191 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376559 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208692 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266922 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419091 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161842 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307726 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227545 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528294 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352574 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199778 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270802 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180482 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308487 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270427 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194278 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226223 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207397 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280985 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035665 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0330372 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0344434 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0436652 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192713 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460714 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424398 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.038083 s, 1.4 GB/s , 0.0339338 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027071 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315336 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310624 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156278 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175312 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173702 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235447 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169526 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151201 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227689 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186425 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258215 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207917 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200624 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226127 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167352 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186754 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202287 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149207 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218742 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218194 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232127 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271108 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0334939 s, 1.6 GB/s , 0.0215879 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187356 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208802 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205871 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459635 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04065 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208906 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200964 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240964 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287419 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167628 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204706 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350739 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184274 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304298 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354805 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420749 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0256708 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0257141 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040548 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456382 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239135 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201594 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156803 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174259 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163958 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030371 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242934 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01877 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018706 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016075 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185419 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270038 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177909 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269377 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307856 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272417 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430833 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154115 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210997 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211876 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160141 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0182735 s, 2.9 GB/s , 0.0278912 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0333877 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232273 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0223334 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197153 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173934 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209316 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183156 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202169 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232525 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280055 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169599 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167855 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331892 s, 1.6 GB/s , 0.0265327 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245798 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163503 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191424 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253243 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153299 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184153 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203591 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265013 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162694 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378948 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017518 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272439 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219167 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282446 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234687 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247643 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337111 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380757 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292908 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172877 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378692 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305659 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0607256 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366435 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04399 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603186 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595191 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282863 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429601 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577983 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478425 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228181 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298402 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520141 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447121 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507512 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229395 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212399 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367408 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0274958 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338718 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325191 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170948 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160904 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170556 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178726 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250333 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229925 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027892 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227434 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261661 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248225 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172565 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018827 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320768 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155847 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021474 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0192969 s, 2.7 GB/s , 0.0196568 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263293 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203746 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224587 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182212 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207132 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236084 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232546 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312645 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192025 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187012 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484357 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0652281 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337077 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461719 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481559 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0405684 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400747 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183309 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341668 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047443 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269971 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160329 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184894 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02561 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169796 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227134 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204688 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287934 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166858 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362113 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263399 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171006 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303242 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270425 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0276403 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460816 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468927 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435267 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268001 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233276 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188684 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188077 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287917 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015642 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171976 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297029 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270656 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180997 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173511 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022571 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176489 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197637 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029554 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247123 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0143785 s, 3.6 GB/s 52428800 bytes (52 MB) copied, 0.0214721 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198842 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015117 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190127 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156256 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179078 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185491 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016105 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241282 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207518 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161739 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01817 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178069 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212055 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253805 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185253 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252989 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169269 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221192 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214644 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448819 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159174 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263224 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255714 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344754 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452178 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376456 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409684 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697054 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312048 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558992 s, 938 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066637 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336116 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0447109 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04245 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486957 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631424 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323762 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159708 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153253 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181456 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183445 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216138 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268563 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256549 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216543 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188213 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170976 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200959 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194452 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328599 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02728 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187725 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177597 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180267 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289051 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429055 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355545 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286645 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318848 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037393 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528332 s, 992 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034242 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0456076 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582629 s, 900 MB/s 52428800 bytes (52 MB) copied, 0.0239944 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317526 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255272 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188553 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164752 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01634 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156752 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025907 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233822 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147153 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215278 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220481 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015516 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171328 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189612 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167926 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149003 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160341 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017023 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209558 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214282 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159464 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214662 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202684 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173221 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188045 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236308 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028827 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017316 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211733 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172872 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161019 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017739 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184458 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191388 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224032 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236807 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0236046 s, 2.2 GB/s , 0.0270766 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254938 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178909 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254787 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176002 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243367 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207763 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252359 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181661 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0366619 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280165 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272149 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157682 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202651 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015623 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165007 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020074 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156556 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156933 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298564 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163561 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245412 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159056 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387361 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200679 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147597 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170341 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211378 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156597 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179098 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159959 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151939 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185796 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199659 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147088 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158421 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221742 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181657 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195044 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224349 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025734 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195654 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368097 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018919 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199136 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244107 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182394 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015451 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208866 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017372 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156211 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212056 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029018 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417041 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.033944 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227959 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258343 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287238 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253068 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232922 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023667 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188449 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025088 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150106 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217299 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182606 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174241 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018881 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183733 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185508 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154966 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215866 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264163 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183605 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204585 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243119 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213596 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184113 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026415 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283927 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257148 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162146 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266666 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172441 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194633 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266814 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205426 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187981 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246363 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213041 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151312 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212384 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203545 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287074 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023173 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212039 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278556 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444581 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.038808 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406322 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437029 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322491 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393928 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336514 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331233 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0219566 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0489867 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327032 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169253 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681718 s, 769 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025452 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172224 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203551 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0287057 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0180739 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188371 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201857 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312598 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354941 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166311 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207607 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349998 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404454 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023731 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173708 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208111 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029925 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315535 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020179 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271447 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021277 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269582 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207693 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.014291 s, 3.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334858 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024443 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277208 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348966 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214394 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026869 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378237 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245464 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056218 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240888 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063655 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020923 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034807 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399724 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159684 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252749 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284593 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336844 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311207 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375618 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628455 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044278 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630475 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871462 s, 602 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0965836 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.108214 s, 484 MB/s , 0.11442 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0950462 s, 552 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127798 s, 410 MB/s 52428800 bytes (52 MB) copied, 0.0980885 s, 535 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106142 s, 494 MB/s 52428800 bytes (52 MB) copied, 0.084093 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099799 s, 525 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0899597 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777205 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332567 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100498 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952292 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101262 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065847 s, 796 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475563 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0510493 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185687 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0791418 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687181 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212022 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202533 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299687 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214382 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289839 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279512 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152758 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343805 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306495 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193545 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193059 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195645 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201318 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277098 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230362 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155483 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215114 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241202 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463987 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282317 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212237 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254563 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580442 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516621 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469576 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287842 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306171 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229324 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194893 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185085 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383003 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248344 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179831 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035813 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029457 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418736 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269165 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213401 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264134 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309544 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022128 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256939 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330456 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03118 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406532 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235791 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201561 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581512 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237436 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155363 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064051 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582489 s, 900 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353766 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0775466 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487903 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355163 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296314 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532438 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391504 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249902 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325391 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629138 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337511 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186405 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340395 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187824 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261567 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0443047 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210722 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0412071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225577 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356851 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171726 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335598 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198679 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020464 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267629 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171851 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168804 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438242 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167274 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347175 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264906 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0259022 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298794 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648106 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544637 s, 963 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472665 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0469772 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025925 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774786 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200569 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678693 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193159 s, 2.7 GB/s , 0.062586 s, 838 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059305 s, 884 MB/s 52428800 bytes (52 MB) copied, 0.0532461 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233019 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169152 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208075 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193691 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456905 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211087 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016397 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168667 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323438 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258016 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284941 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238378 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387269 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334542 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295621 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286557 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150988 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368426 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0624988 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286104 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227466 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0404345 s, 1.3 GB/s , 0.0397626 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358384 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371843 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234009 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200663 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308849 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300068 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175764 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281449 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161106 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200366 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303394 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205921 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372937 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355873 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0282853 s, 1.9 GB/s , 0.0206516 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215418 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193338 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168552 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172413 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166873 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020333 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0138639 s, 3.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221081 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270481 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312921 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026182 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285456 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179929 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388932 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218566 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181924 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543179 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02395 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175261 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0473017 s, 1.1 GB/s , 0.0174419 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269465 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227023 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318974 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345023 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344641 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178093 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313266 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168553 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326035 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0339036 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210408 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383031 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410147 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547727 s, 957 MB/s 52428800 bytes (52 MB) copied, 0.041841 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576536 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680184 s, 771 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608612 s, 861 MB/s 52428800 bytes (52 MB) copied, 0.0448426 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500007 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436545 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710262 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240646 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278021 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430337 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154387 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301263 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194973 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546714 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497983 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192413 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205052 s, 2.6 GB/s , 0.0189786 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430841 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213479 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186009 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237309 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197031 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165439 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246269 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241118 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024029 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372514 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311316 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304226 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217474 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595163 s, 881 MB/s 52428800 bytes (52 MB) copied, 0.0507201 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237363 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293704 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201611 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372705 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250029 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164128 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019589 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199443 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163705 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274591 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214967 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262876 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216159 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263025 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247212 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017297 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298551 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166348 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332869 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187641 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227976 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319273 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0396166 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274422 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574815 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496511 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0390024 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463502 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342874 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0418245 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346424 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297324 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360101 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251566 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209323 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366976 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182758 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185988 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024053 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287199 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352312 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530623 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261856 s, 2.0 GB/s , 0.0504392 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03102 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664451 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527163 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353257 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174992 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379562 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369502 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582495 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600019 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610597 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855058 s, 613 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872556 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477799 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0501308 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443464 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558129 s, 939 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365452 s, 1.4 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533236 s, 983 MB/s , 0.0297956 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422283 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283008 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221488 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200499 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209011 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270426 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268163 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016943 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306968 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274284 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027266 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409552 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228374 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159693 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462931 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257097 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189915 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361751 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425211 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048564 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279272 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0716937 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047399 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772948 s, 678 MB/s 52428800 bytes (52 MB) copied, 0.067325 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678287 s, 773 MB/s 52428800 bytes (52 MB) copied, 0.0412549 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221778 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053523 s, 980 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598801 s, 876 MB/s 52428800 bytes (52 MB) copied, 0.0749504 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469226 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0812149 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645704 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.089801 s, 584 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613199 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478364 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400382 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216419 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225091 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032944 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190598 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188618 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207658 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299221 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026466 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176484 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219474 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309109 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196303 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244831 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292408 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350434 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0312481 s, 1.7 GB/s , 0.0184383 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296574 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340838 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265141 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322932 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447055 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215495 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486846 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229613 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639122 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745857 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426007 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718528 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286181 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460145 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451958 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730593 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535835 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355626 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225615 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204301 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163872 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256675 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168231 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300174 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260982 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211314 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395354 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271097 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416692 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417638 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279528 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310164 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288794 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202976 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353468 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065792 s, 797 MB/s 52428800 bytes (52 MB) copied, 0.0614241 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240105 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325387 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321335 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257894 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362455 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283157 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255252 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193063 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553139 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283047 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520629 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206214 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0529667 s, 990 MB/s 52428800 bytes (52 MB) copied, 0.0175637 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219161 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230692 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355236 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255686 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369649 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240982 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176744 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201566 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0242492 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208339 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0179084 s, 2.9 GB/s , 0.0243806 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187263 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019569 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357164 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325525 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041298 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495242 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196255 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413138 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343148 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184048 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372275 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216633 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200063 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364413 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367243 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502676 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250358 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675788 s, 776 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0509623 s, 1.0 GB/s , 0.0248694 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952994 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0895383 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661872 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805953 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300237 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537775 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423639 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219277 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749981 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582814 s, 900 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0596607 s, 879 MB/s , 0.0449001 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260797 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249277 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333012 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211367 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344106 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242613 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463004 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338457 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323234 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061779 s, 849 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419641 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296621 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401451 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933085 s, 562 MB/s 52428800 bytes (52 MB) copied, 0.055738 s, 941 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0281733 s, 1.9 GB/s , 0.0611135 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725454 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339167 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017533 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265596 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422278 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453152 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288752 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394904 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227164 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031436 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021158 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371223 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417321 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491085 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447358 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377443 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275269 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372778 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257755 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211634 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181091 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516363 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.046964 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370662 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586014 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046052 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599355 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422331 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582565 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034403 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244978 s, 2.1 GB/s , 0.0436214 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316332 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016541 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177214 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207097 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320926 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182474 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033049 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0360212 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308326 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038892 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313685 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405575 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265556 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255392 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03127 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409287 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504046 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653138 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534197 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584972 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622159 s, 843 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767634 s, 683 MB/s 52428800 bytes (52 MB) copied, 0.0754991 s, 694 MB/s 52428800 bytes (52 MB) copied, 0.0597274 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702293 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0940114 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718829 s, 729 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100566 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037372 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0598048 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330957 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356822 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419246 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943022 s, 556 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680981 s, 770 MB/s 52428800 bytes (52 MB) copied, 0.0820015 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190029 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235624 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405562 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017471 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358683 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0382672 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741045 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441381 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021451 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159116 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04241 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314739 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181178 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468091 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183534 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306409 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175561 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020242 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182952 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155085 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171635 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366093 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159809 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192347 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187265 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328437 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257569 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231881 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330929 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024227 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269662 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290257 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251123 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214447 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469376 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254346 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198818 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028362 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387349 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362427 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0267387 s, 2.0 GB/s , 0.0309224 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184059 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177509 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019489 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236258 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204799 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175306 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200437 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530558 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374744 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176935 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276624 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414492 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187327 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230534 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448221 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375076 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355305 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0363486 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379707 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034142 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279884 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179178 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195249 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203883 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021632 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189783 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178987 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213146 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232752 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251111 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304166 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244956 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170878 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234051 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029686 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160637 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255762 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03747 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031205 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489153 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0386404 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706878 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680236 s, 771 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617698 s, 849 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0834175 s, 629 MB/s , 0.0717347 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703906 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08077 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583049 s, 899 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855109 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722206 s, 726 MB/s 52428800 bytes (52 MB) copied, 0.0796445 s, 658 MB/s 52428800 bytes (52 MB) copied, 0.0841681 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337586 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448222 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380956 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049996 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035662 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168299 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229114 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203578 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236557 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191721 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172426 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212485 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347049 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303438 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299445 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264682 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349031 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611799 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818973 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399382 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382891 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031826 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188537 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174866 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168176 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314192 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215678 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029992 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225277 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221316 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017881 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196409 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251916 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021069 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235673 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186631 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391086 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466519 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272706 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273451 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407672 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0325069 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238066 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269666 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028248 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017072 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188817 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022245 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172899 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189155 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021304 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025625 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0188315 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195524 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158473 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162507 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182934 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018583 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174397 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016689 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0322087 s, 1.6 GB/s , 0.0197267 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565678 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400286 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643502 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583357 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703888 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109096 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125476 s, 418 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741839 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127144 s, 412 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752913 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118296 s, 443 MB/s 52428800 bytes (52 MB) copied, 0.15049 s, 348 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102096 s, 514 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12523 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366565 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149531 s, 351 MB/s 52428800 bytes (52 MB) copied, 0.126375 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149798 s, 350 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040141 s, 1.3 GB/s , 0.0801141 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348064 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277589 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226081 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212208 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511185 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384499 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294681 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166964 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417691 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0280383 s, 1.9 GB/s , 0.0281383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185648 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171013 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239189 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260167 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195913 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215611 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192409 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172385 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261643 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251689 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417437 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275253 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373081 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267268 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358677 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226146 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401002 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353681 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289007 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016743 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278838 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270948 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180322 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309057 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210484 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195661 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223875 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188425 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167258 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349219 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182859 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392546 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352063 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185054 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165661 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178788 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244133 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180682 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316429 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196041 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275268 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227718 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233909 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184938 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240918 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160404 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194303 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298194 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199989 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154909 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207063 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185754 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019344 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369172 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220145 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291995 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188723 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346515 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326748 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330605 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380292 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307215 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040572 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465739 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390129 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314496 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699251 s, 750 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239925 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0437272 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035337 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717746 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326156 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527812 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033254 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228372 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362935 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197294 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472389 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299599 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0342168 s, 1.5 GB/s , 0.0173932 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228758 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0253198 s, 2.1 GB/s , 0.0177282 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156383 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195328 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176653 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183907 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163494 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018382 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203262 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020608 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203453 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195317 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231411 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195545 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175386 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173688 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300936 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183737 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208637 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190199 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215223 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285254 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248264 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205013 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247696 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313682 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0217727 s, 2.4 GB/s , 0.0290128 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394616 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193663 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021527 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253544 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184235 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171682 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247041 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170388 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319068 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206579 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341918 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183205 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044171 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0607493 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518749 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857032 s, 612 MB/s 52428800 bytes (52 MB) copied, 0.061785 s, 849 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0987484 s, 531 MB/s , 0.0961312 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627275 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748441 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924064 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748507 s, 700 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460446 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0424777 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415545 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779467 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625576 s, 838 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653572 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736537 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077161 s, 679 MB/s 52428800 bytes (52 MB) copied, 0.069902 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613933 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415043 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0614751 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189744 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178278 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309323 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020454 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020679 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020219 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159096 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190827 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360573 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388514 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451069 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322239 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580442 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520342 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193314 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461465 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699324 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473277 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175203 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355597 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05553 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190366 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04196 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020851 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410748 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221417 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0461194 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0447162 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331597 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288946 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464712 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225426 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409585 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536226 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346433 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289284 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428871 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591056 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198779 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274246 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197548 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507218 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414558 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423435 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270947 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323239 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0913752 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.109391 s, 479 MB/s 52428800 bytes (52 MB) copied, 0.0239064 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429903 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300504 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305763 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587821 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239637 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619184 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037687 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301724 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222602 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191945 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0518415 s, 1.0 GB/s , 0.0274042 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209353 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159529 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024904 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345541 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016384 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395366 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168443 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153686 s, 3.4 GB/s , 0.0159601 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0196999 s, 2.7 GB/s , 0.022128 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173687 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292088 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019585 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183899 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016779 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0272048 s, 1.9 GB/s , 0.0324216 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246106 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261578 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192578 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027989 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214119 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291703 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183824 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313608 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0211876 s, 2.5 GB/s , 0.0210517 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291719 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212093 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502965 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0522574 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485181 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564738 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475509 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525055 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560037 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230728 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319801 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231448 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0426486 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356036 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241882 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587285 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271917 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610517 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822255 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413717 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366235 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602515 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520934 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534118 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545469 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0973292 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385298 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865171 s, 606 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451388 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824284 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931221 s, 563 MB/s 52428800 bytes (52 MB) copied, 0.079019 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664664 s, 789 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0361342 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out , 0.0536217 s, 978 MB/s 52428800 bytes (52 MB) copied, 0.101156 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066014 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891638 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240998 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637866 s, 822 MB/s 52428800 bytes (52 MB) copied, 0.0609755 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203145 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184965 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179445 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264966 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235007 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244816 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247891 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176781 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271549 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029191 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186112 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220153 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213457 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319756 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183944 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016579 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199836 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164248 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177064 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340776 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0399094 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302832 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446591 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0333489 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621604 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709147 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599827 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641873 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580829 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565094 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594625 s, 882 MB/s , 0.0582999 s, 899 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597646 s, 877 MB/s 52428800 bytes (52 MB) copied, 0.067066 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626835 s, 836 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476509 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0434447 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221423 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621242 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559777 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199574 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158973 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183703 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019081 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021765 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197334 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190029 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254279 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165871 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276188 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265982 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165278 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211581 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209255 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213268 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253878 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308291 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255567 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201682 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282757 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269556 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549801 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477996 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204329 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390588 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424193 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226986 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181487 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175528 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320202 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314583 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0373598 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470743 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.031391 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332437 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384956 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381149 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03764 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232825 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0207159 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551611 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396715 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0227772 s, 2.3 GB/s , 0.0488382 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160031 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198821 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0279143 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184433 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190661 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238537 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275963 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181222 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274914 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321608 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029223 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313321 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028765 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212324 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165316 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201262 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244731 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184769 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313241 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436642 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586718 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374336 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264907 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0196843 s, 2.7 GB/s , 0.0392638 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033323 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184069 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028016 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199904 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283348 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185262 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257725 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385943 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194375 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255418 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218433 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376078 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0589183 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016374 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428644 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321157 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215048 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381239 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353709 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242856 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02628 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264287 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019357 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178623 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271016 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016772 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0384723 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201692 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352994 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351024 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343874 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0413816 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285737 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312909 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334536 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272795 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219349 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243007 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029599 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233215 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356254 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446794 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394593 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263204 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380778 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285462 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520196 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190497 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411129 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231039 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185861 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240528 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019841 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329611 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319738 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214175 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028091 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326894 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309832 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242787 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416102 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237141 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418325 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507237 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194419 s, 2.7 GB/s , 0.0245432 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522969 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197655 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324846 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394308 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289893 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187235 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339327 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018106 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396975 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208668 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020958 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225827 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198834 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284874 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210528 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429125 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243113 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576142 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221411 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189945 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040462 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325424 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333768 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0256923 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193553 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191093 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038384 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016888 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03091 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173411 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261356 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0462795 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456835 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511956 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483814 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042944 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347473 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037849 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279543 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204191 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204251 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019221 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351335 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301272 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317942 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380576 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244194 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232366 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435056 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239175 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228563 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235864 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592647 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056057 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331803 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179295 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263633 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176518 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208591 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245082 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246393 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446041 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186127 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025322 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036038 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519268 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536552 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445583 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0429342 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440964 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049351 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296034 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171317 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652415 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047646 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380812 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263873 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338929 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027895 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192319 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206455 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245863 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425544 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02151 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446552 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194032 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217628 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409005 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167877 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673981 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264052 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371653 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501232 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430015 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277458 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489247 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450937 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199158 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196624 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255419 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185334 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198097 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278034 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259407 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290987 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025153 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375781 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464144 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326742 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024364 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038676 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188592 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387788 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560957 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317046 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224061 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484041 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233045 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232244 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0235191 s, 2.2 GB/s , 0.0383409 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338308 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400694 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355442 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198568 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0324924 s, 1.6 GB/s , 0.0217323 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246492 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234044 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0193322 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174267 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236388 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288748 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273844 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283321 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482688 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427349 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462523 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234514 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314717 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317439 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090033 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800042 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241789 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735574 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433714 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0986301 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549852 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295346 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381306 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215092 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535546 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413864 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429157 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346288 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363293 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416914 s, 1.3 GB/s , 0.037861 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244296 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548401 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233583 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020134 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212323 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303089 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353071 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286385 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273179 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253725 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246315 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227176 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178126 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285277 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236336 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347259 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353253 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185281 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302151 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387177 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393802 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339242 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253467 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249412 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184136 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366035 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356046 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591295 s, 887 MB/s 52428800 bytes (52 MB) copied, 0.0671697 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454456 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286228 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811913 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021554 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461687 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055038 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0905077 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543344 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319752 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449943 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358856 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537437 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319964 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241817 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313553 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380664 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411707 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427251 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0229564 s, 2.3 GB/s , 0.0261523 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330096 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341229 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169634 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170313 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217848 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294555 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164899 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286324 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275946 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296888 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213104 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305836 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269912 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395299 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167363 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357435 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028588 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202386 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206043 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318269 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339624 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0302493 s, 1.7 GB/s , 0.025473 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323146 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259544 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257578 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572688 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260824 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418419 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762165 s, 688 MB/s , 0.094632 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349193 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360872 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0687425 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462733 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254517 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726366 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371012 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504882 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0291145 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443805 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0301285 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677797 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728304 s, 720 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206905 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.044515 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352095 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220092 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172809 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173515 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203012 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251435 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272096 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167145 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206948 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218022 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254184 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035512 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059575 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649661 s, 807 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556868 s, 941 MB/s 52428800 bytes (52 MB) copied, 0.0485647 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531418 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711337 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929844 s, 564 MB/s , 0.0778532 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0938826 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465397 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583887 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0908534 s, 577 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.10918 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100249 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068481 s, 766 MB/s 52428800 bytes (52 MB) copied, 0.0878079 s, 597 MB/s 52428800 bytes (52 MB) copied, 0.0780982 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554244 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654293 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759375 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434164 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569326 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348231 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424868 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800066 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696776 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762701 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180667 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017409 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016475 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199159 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278578 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0309806 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403519 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047822 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365887 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352379 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440627 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0341941 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467336 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037544 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196194 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175428 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309675 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300256 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019701 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155073 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288534 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216254 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175931 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174983 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218326 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192772 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259318 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225532 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019744 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457018 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186519 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286936 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186191 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247345 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228709 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234879 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294868 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206204 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01901 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232517 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181152 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022015 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246115 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209873 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0222765 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222485 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411183 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239676 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215447 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176576 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302323 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271519 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302958 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215749 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228476 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214652 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179836 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165436 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272815 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341649 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341174 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253292 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450591 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235434 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238641 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030853 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300725 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211895 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222361 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213295 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213478 s, 2.5 GB/s , 0.0256422 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219361 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185723 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205499 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287107 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018201 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223136 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433666 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441359 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261068 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435394 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0552792 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345676 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345122 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017915 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455341 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324276 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199786 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203698 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196121 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181775 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182384 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182442 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318292 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341831 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205641 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206381 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269963 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283762 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194149 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335192 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179263 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211855 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187969 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283113 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332969 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290005 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288112 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351121 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207961 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338049 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277044 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167092 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384759 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175061 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189799 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431035 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0580829 s, 903 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025379 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0486192 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040381 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0337457 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253156 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164636 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250193 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021779 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169893 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245878 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0257716 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152486 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233131 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183496 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357467 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244636 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203408 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231955 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158315 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033237 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331313 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228025 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210211 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168033 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222874 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197854 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176448 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188948 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193418 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187188 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180788 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232198 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171972 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198691 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211548 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203277 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205267 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181651 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216445 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176982 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176013 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213648 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015902 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185657 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328439 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175617 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188006 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163613 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266139 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.018135 s, 2.9 GB/s , 0.0196532 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193319 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016094 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153504 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176971 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176641 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253343 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161775 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177724 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200834 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189661 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212314 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248348 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205713 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236669 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189826 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177899 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367553 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246111 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0303738 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162295 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196665 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268552 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0146708 s, 3.6 GB/s , 0.0173546 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018605 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197409 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179377 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180803 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204021 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016107 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198932 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250343 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170259 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182498 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292955 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.017038 s, 3.1 GB/s , 0.0253801 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163451 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194257 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176256 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202596 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019628 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166652 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231525 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208689 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174916 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196742 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193712 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197564 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162015 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178918 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178331 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244534 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190569 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171052 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505432 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232507 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04124 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348896 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0386701 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456525 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0430602 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289191 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269509 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0315546 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0184288 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202556 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180307 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299225 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359278 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279749 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022284 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367557 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369199 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290102 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318608 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171871 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206862 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225073 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226904 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238942 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389203 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0350937 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560053 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447711 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430566 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673032 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0493571 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599734 s, 874 MB/s 52428800 bytes (52 MB) copied, 0.0491687 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400606 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561003 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054349 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498419 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330477 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048015 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600545 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568645 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531366 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352635 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493283 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457129 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465743 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0380316 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332193 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256243 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183419 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0179828 s, 2.9 GB/s , 0.0193728 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325456 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.021128 s, 2.5 GB/s , 0.021088 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272156 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265921 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331574 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585777 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385773 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034752 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622386 s, 842 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352349 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0437389 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688349 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323917 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639256 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538393 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483339 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454503 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218328 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038594 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354633 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172991 s, 3.0 GB/s , 0.0246989 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164139 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213789 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02661 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253889 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194283 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0277778 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269042 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321662 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462988 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473853 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596615 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432224 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455115 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535425 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480409 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054735 s, 958 MB/s 52428800 bytes (52 MB) copied, 0.0433768 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0286345 s, 1.8 GB/s , 0.0339314 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047028 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592508 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357553 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394653 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234364 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249201 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163347 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212861 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245269 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234815 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211491 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204428 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368565 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425078 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0345293 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032521 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552712 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05901 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335886 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378453 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907349 s, 578 MB/s 52428800 bytes (52 MB) copied, 0.0288041 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0930306 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554695 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389945 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377614 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346471 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462047 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259025 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490978 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644396 s, 814 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625565 s, 838 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031429 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0455127 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0414865 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389325 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396684 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375781 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227487 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018818 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216203 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195574 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203847 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180542 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424718 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313607 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319619 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454237 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218471 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237063 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192696 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174172 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179818 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230315 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244419 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018377 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288647 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024261 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317778 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223607 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290065 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284422 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173505 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282508 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202307 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266623 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259976 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266096 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066669 s, 786 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0459924 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332273 s, 1.6 GB/s 50+0 records in 50+0 records out , 0.0480152 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592306 s, 885 MB/s 52428800 bytes (52 MB) copied, 0.0503849 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316815 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512679 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365081 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.076182 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613106 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383949 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610972 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432319 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038893 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402751 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0884109 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966026 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107074 s, 490 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813375 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183207 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237941 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026768 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261809 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348455 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361242 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261783 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175672 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485641 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298753 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0587622 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545343 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257546 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337105 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398988 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306726 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016333 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172225 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327084 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456893 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213239 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326744 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192886 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247648 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225822 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317172 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336181 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0549922 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056569 s, 927 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056543 s, 927 MB/s 52428800 bytes (52 MB) copied, 0.0509713 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528644 s, 992 MB/s 52428800 bytes (52 MB) copied, 0.0693816 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622235 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901506 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563017 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385185 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059268 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044549 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203467 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947324 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441839 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273976 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605276 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178826 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246788 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214826 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022422 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177275 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305522 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206705 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196777 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194771 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253427 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177974 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170159 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228983 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293983 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313835 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208302 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287861 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276862 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0387609 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352095 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390608 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228287 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.023487 s, 2.2 GB/s , 0.0369696 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168713 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162069 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240596 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176098 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020902 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035012 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171215 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250337 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220634 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171569 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173283 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330714 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020039 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0205245 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260018 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174878 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588033 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480763 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0645544 s, 812 MB/s 52428800 bytes (52 MB) copied, 0.0470981 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572513 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762164 s, 688 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0768584 s, 682 MB/s , 0.065134 s, 805 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774825 s, 677 MB/s , 0.0680766 s, 770 MB/s 52428800 bytes (52 MB) copied, 0.0744522 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585062 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681019 s, 770 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708376 s, 740 MB/s 52428800 bytes (52 MB) copied, 0.0393343 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700789 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447742 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355755 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230186 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606932 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033669 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628508 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182195 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342036 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154316 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227613 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175703 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259511 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191367 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378915 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019192 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416465 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216429 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330163 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173191 s, 3.0 GB/s , 0.0211661 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022472 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319031 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267417 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372825 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265946 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220688 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169624 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021317 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177068 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032989 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03042 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242273 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225058 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235526 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198772 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287208 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289942 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0380492 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271911 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277051 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397262 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191889 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181371 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167804 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246277 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349635 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178581 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198124 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606817 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367545 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0329157 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166799 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256562 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229071 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211597 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209672 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182239 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016538 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180102 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0373191 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152814 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171801 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184871 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183201 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229112 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028186 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308673 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0220476 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254945 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383535 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290005 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287882 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029034 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197561 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165143 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290155 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173906 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178058 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265764 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018336 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241793 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265961 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290701 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197062 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177194 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319813 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373605 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199934 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018288 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218178 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032463 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367332 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163201 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154213 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200871 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266259 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175652 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169246 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233668 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196715 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158683 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0149211 s, 3.5 GB/s 52428800 bytes (52 MB) copied, 0.0169088 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188681 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169072 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158741 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025297 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180369 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257637 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029487 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183345 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216506 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243745 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017139 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388193 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020105 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257411 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017628 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156316 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179717 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208015 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254834 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260273 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265473 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220169 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222825 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229897 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180352 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174777 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178089 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179381 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021822 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231812 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273422 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023112 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233016 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303471 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205859 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266022 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280407 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017029 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193527 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249269 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190372 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285861 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465673 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345339 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334251 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312225 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559362 s, 937 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384879 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483278 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735537 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342582 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258351 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0427015 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639956 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488406 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.027534 s, 1.9 GB/s , 0.0188458 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209375 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174086 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210497 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177371 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219771 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198666 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182292 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191009 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307849 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200793 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214257 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310225 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192439 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524304 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207628 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234517 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194458 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251899 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171753 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176506 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224551 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273568 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201417 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231518 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225254 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261286 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331894 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171639 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251793 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271606 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.02168 s, 2.4 GB/s , 0.0297464 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244888 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197297 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193138 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213731 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254407 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210226 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279405 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189566 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236497 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178393 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197524 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225492 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0349567 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0314443 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247051 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0403486 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388399 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0384126 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211301 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517219 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485251 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.047647 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285588 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531843 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519376 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.032996 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530353 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159913 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542066 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172124 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192346 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197374 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206751 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255685 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155614 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299473 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355273 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342832 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523609 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236264 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474516 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340853 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332093 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421901 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416429 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428679 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529231 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525636 s, 997 MB/s 52428800 bytes (52 MB) copied, 0.0273354 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428769 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202885 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383544 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308257 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347362 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328108 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319057 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423736 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386109 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505663 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0576419 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477007 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671575 s, 781 MB/s 52428800 bytes (52 MB) copied, 0.0469624 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209235 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279348 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427247 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328399 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164274 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199389 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210231 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237183 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025183 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268638 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0327194 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346845 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195853 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328485 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274489 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206403 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207208 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587953 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642298 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361893 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416151 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366807 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0603675 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305255 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289163 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231421 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0255411 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296709 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181391 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227204 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224089 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188587 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162557 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169601 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210653 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186182 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233316 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027671 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293197 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248156 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0473829 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450557 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273843 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253537 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195449 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434935 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018416 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019904 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.029125 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0184732 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422906 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351436 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0223564 s, 2.3 GB/s , 0.0293045 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213426 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150344 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166736 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191105 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291723 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259453 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357181 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218489 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376275 s, 1.4 GB/s , 0.0368312 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290153 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283901 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226132 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226425 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353766 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249827 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367578 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193479 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216815 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217519 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022813 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166044 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191469 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275052 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162253 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193488 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167386 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150759 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158353 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215526 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205378 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213408 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379956 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415859 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417016 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371787 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333457 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345285 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335939 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243432 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0532685 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455342 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223914 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258819 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197389 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191214 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192143 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02558 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179742 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166478 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214847 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193729 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266829 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0407942 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278018 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500832 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434655 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278317 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279757 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205627 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187717 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168311 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226667 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178814 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208098 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226625 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030729 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0258227 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0178247 s, 2.9 GB/s , 0.0282288 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020813 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251096 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305644 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029327 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184506 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023385 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268868 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189437 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320431 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018405 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184346 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291324 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292369 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386641 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0546219 s, 960 MB/s 52428800 bytes (52 MB) copied, 0.0194927 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492324 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380415 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01684 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257588 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206402 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180478 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183606 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212632 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205766 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266156 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295696 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233051 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028007 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225365 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276212 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199722 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292766 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222234 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423611 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02391 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709982 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561292 s, 934 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634528 s, 826 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041076 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0375789 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0626397 s, 837 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552043 s, 950 MB/s 52428800 bytes (52 MB) copied, 0.0606534 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484676 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326523 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346815 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380559 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.067492 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056756 s, 924 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.029655 s, 1.8 GB/s , 0.0406121 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026328 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017358 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187802 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217813 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201017 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176608 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178196 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224082 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174832 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195354 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0399511 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322244 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200431 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185373 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148137 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157563 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215593 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179135 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224188 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167097 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02504 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326996 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302802 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317872 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495937 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285081 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417949 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593683 s, 883 MB/s 52428800 bytes (52 MB) copied, 0.0354082 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425155 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250575 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195055 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489841 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439272 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306296 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176546 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274571 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0433697 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245932 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202529 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200885 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171602 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225449 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193094 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022075 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304977 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191603 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257313 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177122 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190854 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169537 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219228 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255905 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022423 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031395 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398508 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231004 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228644 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206093 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374386 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213961 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204878 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205483 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166588 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439114 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241465 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194927 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268597 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186641 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244767 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266183 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030335 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177073 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283723 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166969 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257782 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451829 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186755 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400139 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466198 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.049936 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454517 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357039 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252459 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566659 s, 925 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056269 s, 932 MB/s 52428800 bytes (52 MB) copied, 0.0390027 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774293 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647976 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506659 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434723 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033821 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336699 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230966 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214619 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177131 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170819 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183472 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201855 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215018 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197004 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180305 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205883 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195863 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225448 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018586 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259192 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342445 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290481 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183956 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232441 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170722 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176743 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152947 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180872 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172959 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199021 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028455 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264783 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242406 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282177 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560816 s, 935 MB/s 52428800 bytes (52 MB) copied, 0.0359556 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381536 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423059 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0341405 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242834 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419228 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.033737 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205709 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319177 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463134 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0225567 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307898 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374974 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283594 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0227926 s, 2.3 GB/s , 0.0400043 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471467 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395851 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223766 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208874 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240024 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260812 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287337 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459789 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358256 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377461 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596094 s, 880 MB/s 52428800 bytes (52 MB) copied, 0.0778805 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294678 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373224 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367249 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427955 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182103 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415863 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214569 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183712 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318602 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337081 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325422 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327225 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315075 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465693 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484233 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0426666 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305178 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036686 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0283479 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675405 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800172 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0842 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050518 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270171 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029764 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088119 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102276 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590542 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263418 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018616 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174671 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197236 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169469 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235042 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169436 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232715 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205437 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025803 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275267 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277754 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183377 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197926 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197675 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221115 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210949 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02067 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161228 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319358 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186839 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020807 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230802 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301841 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167111 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257077 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217108 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385563 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431266 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549268 s, 955 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690302 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318941 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395139 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0747377 s, 702 MB/s 52428800 bytes (52 MB) copied, 0.0630855 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487338 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05347 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575003 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205387 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508569 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031765 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0431895 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480908 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024245 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021795 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184269 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174888 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205757 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199001 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202333 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337327 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313605 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283057 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440631 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363188 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022955 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0632621 s, 829 MB/s , 0.0412241 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389459 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456085 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516707 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193769 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232652 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027964 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259731 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023776 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031634 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227884 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188141 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023289 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229069 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017787 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228282 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316699 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175497 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500292 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0381941 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291773 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786641 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327485 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638324 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576919 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415911 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689854 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505477 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284273 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398672 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340427 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032703 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178359 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208794 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303641 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194888 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205692 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267555 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201775 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244551 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369712 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391973 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295439 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461936 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440737 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208802 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308935 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361445 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030283 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025332 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168247 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458239 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377804 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349908 s, 1.5 GB/s , 0.0263884 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228112 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222463 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336569 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174146 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299204 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398571 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320199 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278532 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0446716 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199124 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281658 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546673 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183478 s, 2.9 GB/s , 0.0180711 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316114 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291047 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203499 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190636 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288689 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307598 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317925 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017201 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268289 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283903 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220012 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028259 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164792 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345706 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194031 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199036 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194818 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608429 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191333 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529912 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360234 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374893 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.085795 s, 611 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552544 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401334 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177776 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352461 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497065 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442959 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196383 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303628 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397215 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403731 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029472 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239295 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362257 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0484118 s, 1.1 GB/s , 0.0403632 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289951 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339046 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410939 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412244 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338533 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289482 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022814 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321679 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159166 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346137 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428948 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235318 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233218 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557987 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230039 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489702 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542984 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664874 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0792409 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277557 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064212 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775586 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307177 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282776 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269241 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552162 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198563 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263288 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177832 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218919 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236193 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179827 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265709 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213246 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198142 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426591 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261415 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185581 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210928 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217887 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179288 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344905 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0429625 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406317 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293568 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446064 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530589 s, 988 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453427 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258608 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.061764 s, 849 MB/s 52428800 bytes (52 MB) copied, 0.0373707 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028336 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418391 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0185069 s, 2.8 GB/s , 0.0203534 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222461 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417443 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269964 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271635 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285071 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201288 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382624 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464884 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371331 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019302 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453745 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584695 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446716 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427444 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758649 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554754 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782245 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565242 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606817 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886481 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056605 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281757 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317323 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234515 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021886 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366179 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454157 s, 1.2 GB/s , 0.0411877 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618892 s, 847 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380851 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0389234 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675862 s, 776 MB/s 52428800 bytes (52 MB) copied, 0.0473196 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497412 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304937 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500143 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557443 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036298 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181449 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061813 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163029 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210631 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289753 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174734 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510403 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045122 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056297 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186299 s, 2.8 GB/s , 0.0443279 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178275 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329099 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272912 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265984 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250243 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0217016 s, 2.4 GB/s , 0.0218089 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049107 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521894 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211461 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286949 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293455 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290464 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418272 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299687 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0607798 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679068 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690563 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442002 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.087738 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907418 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696704 s, 753 MB/s 52428800 bytes (52 MB) copied, 0.0772634 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137281 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886735 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0808701 s, 648 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595886 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0911186 s, 575 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123868 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119389 s, 439 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116969 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497573 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0997217 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182904 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15559 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155758 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932669 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212901 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088437 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107903 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452451 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633083 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164302 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040137 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152883 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314677 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550809 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788738 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.087559 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579782 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112998 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102545 s, 511 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11202 s, 468 MB/s 52428800 bytes (52 MB) copied, 0.0926709 s, 566 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131789 s, 398 MB/s 52428800 bytes (52 MB) copied, 0.0991447 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106043 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122271 s, 429 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187533 s, 280 MB/s 52428800 bytes (52 MB) copied, 0.11436 s, 458 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181519 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.190782 s, 275 MB/s , 0.139934 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118566 s, 442 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158231 s, 331 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15752 s, 333 MB/s 52428800 bytes (52 MB) copied, 0.100858 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.0927949 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776017 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088626 s, 592 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166563 s, 315 MB/s 52428800 bytes (52 MB) copied, 0.117634 s, 446 MB/s 52428800 bytes (52 MB) copied, 0.14202 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651575 s, 805 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718346 s, 730 MB/s 52428800 bytes (52 MB) copied, 0.0362703 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14252 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553437 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715707 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416027 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750404 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689454 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361683 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362142 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129478 s, 405 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821451 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292529 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0191257 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231386 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176292 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358137 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255859 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191799 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436033 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220259 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281807 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218134 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424012 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0238617 s, 2.2 GB/s , 0.0448933 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165405 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203139 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250315 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308979 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178594 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425934 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384582 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035483 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459749 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570572 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490763 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066864 s, 784 MB/s , 0.0754612 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423146 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440667 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061481 s, 853 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314684 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0406054 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255832 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276837 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279253 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576217 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519502 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220491 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392338 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237416 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221764 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164945 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244996 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238733 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319164 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032215 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192916 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281861 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412627 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297743 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324436 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294318 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230299 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042133 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468266 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273906 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256929 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210219 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270177 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204474 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321227 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194444 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482915 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038649 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02203 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164361 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217284 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171629 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257183 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289351 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017761 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453348 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176865 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395492 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482577 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268397 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031268 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525269 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042426 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554897 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514953 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540432 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449537 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593359 s, 884 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162436 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656316 s, 799 MB/s 52428800 bytes (52 MB) copied, 0.0791694 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418818 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582261 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218282 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169433 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224694 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204209 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280291 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148669 s, 3.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368268 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0430572 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422318 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171732 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230336 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358376 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037755 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230615 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479063 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321051 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323749 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518665 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504893 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325946 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341314 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629075 s, 833 MB/s , 0.0270172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249269 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373625 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0280989 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155368 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186376 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328864 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374969 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396329 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340062 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302746 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336756 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042667 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330997 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614248 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806823 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616936 s, 850 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864937 s, 606 MB/s 52428800 bytes (52 MB) copied, 0.106611 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0979066 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853474 s, 614 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133569 s, 393 MB/s 52428800 bytes (52 MB) copied, 0.113022 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130754 s, 401 MB/s 52428800 bytes (52 MB) copied, 0.143314 s, 366 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154548 s, 339 MB/s 52428800 bytes (52 MB) copied, 0.187705 s, 279 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148811 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142688 s, 367 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196128 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.118934 s, 441 MB/s 50+0 records in 50+0 records out , 0.142316 s, 368 MB/s 52428800 bytes (52 MB) copied, 0.22559 s, 232 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134387 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.192626 s, 272 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975057 s, 538 MB/s 52428800 bytes (52 MB) copied, 0.175375 s, 299 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170985 s, 307 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214656 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14193 s, 369 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193066 s, 272 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.179975 s, 291 MB/s , 0.148109 s, 354 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131292 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267267 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0747978 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15584 s, 336 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109963 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985273 s, 532 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127963 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775643 s, 676 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107499 s, 488 MB/s 52428800 bytes (52 MB) copied, 0.0624037 s, 840 MB/s 52428800 bytes (52 MB) copied, 0.205858 s, 255 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708331 s, 740 MB/s 52428800 bytes (52 MB) copied, 0.0391206 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632939 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658045 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310232 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0784782 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830686 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398121 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657452 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268003 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0426073 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465075 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335321 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0403852 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404593 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0332456 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0238426 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210737 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266579 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286132 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267662 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0264049 s, 2.0 GB/s , 0.0665821 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430013 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464123 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383626 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175908 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344108 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210439 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495185 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175367 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244614 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249221 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420139 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0731243 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904395 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577872 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277788 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618088 s, 848 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261185 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395582 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0468088 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538915 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477329 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150708 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510303 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177975 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186594 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0276623 s, 1.9 GB/s , 0.0316139 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190794 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223357 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188557 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0375899 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274697 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294729 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516085 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437715 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734354 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796015 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512423 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749702 s, 699 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729109 s, 719 MB/s 52428800 bytes (52 MB) copied, 0.0712569 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486663 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578975 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231765 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647116 s, 810 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668079 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836378 s, 627 MB/s 52428800 bytes (52 MB) copied, 0.104466 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104163 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106889 s, 490 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054824 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683185 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138349 s, 379 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703891 s, 745 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153911 s, 341 MB/s 52428800 bytes (52 MB) copied, 0.102905 s, 509 MB/s 52428800 bytes (52 MB) copied, 0.0756745 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140719 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056359 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129253 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177926 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996398 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680241 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108435 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066039 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146162 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648776 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669904 s, 783 MB/s , 0.0510879 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238085 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424399 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477806 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229314 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184625 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232113 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386065 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220358 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362348 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044046 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248415 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320026 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230807 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015686 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298951 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235603 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417169 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172162 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254474 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245044 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404632 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379563 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219903 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211479 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203586 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168771 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440818 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263472 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297622 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228144 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286209 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182776 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252863 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301396 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319188 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216133 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476205 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335361 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231686 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738904 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511765 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067863 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465639 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467085 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455239 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159917 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423311 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426873 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398682 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346363 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267304 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708355 s, 740 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0823505 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717693 s, 731 MB/s 52428800 bytes (52 MB) copied, 0.0718214 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022267 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029311 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245145 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438129 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498475 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254204 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168562 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257772 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228968 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311451 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334055 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333745 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042891 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.062684 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222198 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602585 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761306 s, 689 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453776 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0394221 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611997 s, 857 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740916 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376902 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0636509 s, 824 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307307 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.11004 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0790942 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204841 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104761 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539771 s, 971 MB/s 50+0 records in 50+0 records out , 0.0830597 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592036 s, 886 MB/s 52428800 bytes (52 MB) copied, 0.0812496 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0751005 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240482 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168721 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160676 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159116 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281573 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418538 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0965304 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126405 s, 415 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667834 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959526 s, 546 MB/s 52428800 bytes (52 MB) copied, 0.12493 s, 420 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159163 s, 329 MB/s 52428800 bytes (52 MB) copied, 0.0831338 s, 631 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117714 s, 445 MB/s 52428800 bytes (52 MB) copied, 0.129308 s, 405 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959937 s, 546 MB/s 52428800 bytes (52 MB) copied, 0.15594 s, 336 MB/s 52428800 bytes (52 MB) copied, 0.186507 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139483 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118659 s, 442 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948263 s, 553 MB/s 52428800 bytes (52 MB) copied, 0.0789241 s, 664 MB/s 52428800 bytes (52 MB) copied, 0.125245 s, 419 MB/s 52428800 bytes (52 MB) copied, 0.20308 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104397 s, 502 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168019 s, 312 MB/s 52428800 bytes (52 MB) copied, 0.0905004 s, 579 MB/s 52428800 bytes (52 MB) copied, 0.0846741 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183261 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942748 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551741 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105305 s, 498 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180884 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0929794 s, 564 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.105791 s, 496 MB/s , 0.0993666 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703418 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394732 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198972 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196691 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729372 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673198 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269587 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257097 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243503 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366498 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210489 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216612 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356867 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505758 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040626 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304904 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438941 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261458 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371076 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604958 s, 867 MB/s 52428800 bytes (52 MB) copied, 0.0426493 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350283 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954067 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466296 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339043 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352312 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788165 s, 665 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541846 s, 968 MB/s 52428800 bytes (52 MB) copied, 0.0477964 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544389 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545319 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477156 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586001 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433964 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477235 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427923 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256386 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509601 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268285 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274548 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0658294 s, 796 MB/s , 0.0164147 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168878 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388922 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016204 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027492 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308492 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322142 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415239 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212084 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426194 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345657 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377246 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056729 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389796 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230295 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032437 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384507 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266397 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191779 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281613 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198786 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171294 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431551 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270049 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375598 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289129 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352765 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298725 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315078 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546962 s, 959 MB/s 52428800 bytes (52 MB) copied, 0.0288358 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335431 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191378 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341843 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215903 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290657 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376339 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216702 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253426 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315976 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497049 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274227 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373978 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034035 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668406 s, 784 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048671 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688332 s, 762 MB/s 52428800 bytes (52 MB) copied, 0.0479785 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0748293 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502772 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.084142 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459535 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678033 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0973036 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070237 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167175 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333202 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032046 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182011 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210245 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021294 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319086 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212903 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427716 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308916 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452725 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407455 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365412 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317892 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252521 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060567 s, 866 MB/s 52428800 bytes (52 MB) copied, 0.0508691 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585137 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901844 s, 581 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108664 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0998478 s, 525 MB/s 52428800 bytes (52 MB) copied, 0.0994098 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121073 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103661 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149558 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164348 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139143 s, 377 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133586 s, 392 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112212 s, 467 MB/s , 0.118304 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200763 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103475 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134183 s, 391 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177472 s, 295 MB/s 52428800 bytes (52 MB) copied, 0.162137 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982846 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0995914 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09104 s, 576 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128678 s, 407 MB/s 52428800 bytes (52 MB) copied, 0.065336 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111755 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129462 s, 405 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749246 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837504 s, 626 MB/s 52428800 bytes (52 MB) copied, 0.0755552 s, 694 MB/s 52428800 bytes (52 MB) copied, 0.0652105 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825896 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672783 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691439 s, 758 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216778 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336811 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0737737 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223553 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334889 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591718 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454577 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603724 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327078 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165277 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203234 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251493 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231901 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243114 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177515 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181977 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250204 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038229 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247892 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183456 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298987 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225507 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210052 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407231 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378264 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0555703 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249419 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232079 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544184 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221554 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288548 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646175 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657395 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101749 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814662 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806086 s, 650 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11319 s, 463 MB/s 52428800 bytes (52 MB) copied, 0.0671325 s, 781 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622769 s, 842 MB/s 52428800 bytes (52 MB) copied, 0.102638 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056061 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710893 s, 738 MB/s 52428800 bytes (52 MB) copied, 0.0917543 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269144 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099101 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904724 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136778 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120645 s, 435 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133663 s, 392 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643854 s, 814 MB/s 52428800 bytes (52 MB) copied, 0.0925797 s, 566 MB/s 52428800 bytes (52 MB) copied, 0.0491885 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108043 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138468 s, 379 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109783 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12065 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12317 s, 426 MB/s 52428800 bytes (52 MB) copied, 0.0410664 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140626 s, 373 MB/s 52428800 bytes (52 MB) copied, 0.0579366 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164725 s, 318 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876555 s, 598 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0923645 s, 568 MB/s , 0.0789713 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136734 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.136056 s, 385 MB/s 52428800 bytes (52 MB) copied, 0.0943005 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654748 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529525 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080409 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640594 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053552 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488748 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229568 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227634 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311508 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180116 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408902 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342225 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268613 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232948 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184738 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182243 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204654 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199219 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211247 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406952 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533125 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584786 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676569 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687147 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424222 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.096028 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0925891 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413611 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081644 s, 642 MB/s 52428800 bytes (52 MB) copied, 0.056921 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13055 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697978 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769401 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270622 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412907 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078096 s, 671 MB/s 52428800 bytes (52 MB) copied, 0.10283 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664288 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983509 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049294 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0469802 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0210354 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296763 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080857 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024359 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218617 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314539 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172434 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234425 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0283163 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333674 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155795 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338393 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414992 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0529267 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406323 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234986 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370257 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411621 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244042 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292814 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461414 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0573529 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400026 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366329 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306302 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444897 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515896 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299938 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373247 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362543 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297267 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418503 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289839 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247831 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189894 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300945 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034439 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222614 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296694 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356668 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171636 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227526 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449379 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0191904 s, 2.7 GB/s , 0.0431628 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249664 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249012 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173616 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036996 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370137 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351025 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200338 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308616 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260717 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191997 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02363 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198037 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194152 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265967 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268159 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258537 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253378 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289796 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246924 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268594 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250423 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0512193 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042513 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332675 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317663 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786172 s, 667 MB/s 52428800 bytes (52 MB) copied, 0.0781317 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766546 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671551 s, 781 MB/s 52428800 bytes (52 MB) copied, 0.0716039 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103045 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0984439 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0998705 s, 525 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776333 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642117 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620684 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0951785 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394881 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064022 s, 819 MB/s 52428800 bytes (52 MB) copied, 0.047493 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0915415 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332088 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273794 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345296 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263218 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250555 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334162 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260862 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174145 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271217 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220941 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230025 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364723 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03721 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154768 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233528 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035344 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0478607 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370239 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382894 s, 1.4 GB/s , 0.0633844 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328815 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627151 s, 836 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433377 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0351792 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848894 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284316 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0443288 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551627 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034359 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880958 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310741 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424843 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179206 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074976 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737232 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267786 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708032 s, 740 MB/s 52428800 bytes (52 MB) copied, 0.038139 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05444 s, 963 MB/s 52428800 bytes (52 MB) copied, 0.0556289 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314022 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034225 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888483 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396579 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180602 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251062 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183549 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223528 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301597 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226024 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0186616 s, 2.8 GB/s , 0.0241312 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167125 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246614 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403569 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344374 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196403 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398989 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347482 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265931 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041979 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359073 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360018 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269555 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187843 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349841 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0313446 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513254 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368114 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478225 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342409 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455673 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633646 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763887 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451182 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318596 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417268 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361965 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05923 s, 885 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356782 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205051 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0421097 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198319 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388845 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251619 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225603 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260102 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166213 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189294 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197787 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348519 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0347985 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518449 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591291 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533475 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360946 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330892 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540438 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405634 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486434 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0263664 s, 2.0 GB/s , 0.0265687 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350914 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211636 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231098 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164347 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156649 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293796 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234288 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161474 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313692 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205186 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151639 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283732 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195903 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182099 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153631 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231972 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218162 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02077 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177072 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251188 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031119 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244984 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699596 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597873 s, 877 MB/s 52428800 bytes (52 MB) copied, 0.0599113 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632662 s, 829 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735951 s, 712 MB/s 52428800 bytes (52 MB) copied, 0.0676817 s, 775 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043619 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353214 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.066422 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040901 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092542 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282622 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242301 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219908 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360297 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359348 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666291 s, 787 MB/s 52428800 bytes (52 MB) copied, 0.0542343 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350662 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299726 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390336 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349951 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388287 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272616 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170384 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159028 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237162 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03166 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300678 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300383 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244345 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270154 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386246 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237414 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342249 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161998 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181706 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178678 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361946 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359444 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028751 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327154 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382137 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260091 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164997 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361751 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432625 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339018 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396745 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351435 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569282 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372896 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574916 s, 912 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625104 s, 839 MB/s 52428800 bytes (52 MB) copied, 0.071721 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566958 s, 925 MB/s 52428800 bytes (52 MB) copied, 0.07383 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590329 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0962154 s, 545 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546088 s, 960 MB/s 52428800 bytes (52 MB) copied, 0.0789255 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449025 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051532 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351368 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272704 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641821 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484403 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587888 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836186 s, 627 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339265 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570506 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470421 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583703 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233849 s, 2.2 GB/s , 0.0256512 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022035 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290886 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225429 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485428 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039758 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03495 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474366 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468253 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372936 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381724 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484254 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311795 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481057 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626886 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322577 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789004 s, 664 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620119 s, 845 MB/s 52428800 bytes (52 MB) copied, 0.0569343 s, 921 MB/s 52428800 bytes (52 MB) copied, 0.0821135 s, 638 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078217 s, 670 MB/s 52428800 bytes (52 MB) copied, 0.144033 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410915 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.145903 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983922 s, 533 MB/s 52428800 bytes (52 MB) copied, 0.11593 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137995 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746229 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982507 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111989 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111268 s, 471 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989363 s, 530 MB/s 52428800 bytes (52 MB) copied, 0.0738324 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115429 s, 454 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700996 s, 748 MB/s 52428800 bytes (52 MB) copied, 0.115699 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102418 s, 512 MB/s 52428800 bytes (52 MB) copied, 0.0803565 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121325 s, 432 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599694 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0968038 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444299 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328902 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575774 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498152 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040662 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307606 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487265 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415696 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225539 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365186 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248147 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335365 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314045 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0596536 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843695 s, 621 MB/s 52428800 bytes (52 MB) copied, 0.0413946 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294589 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671708 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330668 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308974 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186812 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281323 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205858 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258621 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276912 s, 1.9 GB/s , 0.0426995 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609249 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863838 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390862 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539041 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406504 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091381 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106546 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510966 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824409 s, 636 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428044 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528551 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482018 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400456 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699448 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057374 s, 914 MB/s 52428800 bytes (52 MB) copied, 0.103385 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466837 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524398 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681126 s, 770 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1349 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220245 s, 2.4 GB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0441943 s, 1.2 GB/s , 0.0698889 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447079 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187456 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248445 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246177 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163622 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175399 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157934 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181951 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0191444 s, 2.7 GB/s , 0.0194191 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179326 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188561 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222703 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350476 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212699 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600605 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190784 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190287 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230097 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326269 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250747 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442059 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305654 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274897 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726144 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441235 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357759 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249965 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675983 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502937 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337041 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0655632 s, 800 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0747915 s, 701 MB/s 52428800 bytes (52 MB) copied, 0.0405295 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310066 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0648658 s, 808 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120832 s, 434 MB/s 52428800 bytes (52 MB) copied, 0.055435 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430591 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614203 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820985 s, 639 MB/s , 0.0548824 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403604 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449248 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534333 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387033 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493035 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461929 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249107 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507912 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324601 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595279 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268064 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381541 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358635 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344754 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0971104 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040361 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177808 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502401 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207644 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520513 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249137 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277825 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184783 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557156 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376725 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028847 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163233 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337059 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245014 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316381 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026094 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223529 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345078 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034902 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703076 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04377 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452907 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481541 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102646 s, 511 MB/s 52428800 bytes (52 MB) copied, 0.0636755 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796041 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851054 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288712 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052148 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0757381 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628108 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137177 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451833 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885384 s, 592 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08144 s, 644 MB/s 52428800 bytes (52 MB) copied, 0.0409105 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634922 s, 826 MB/s , 0.0614821 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555283 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312305 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417252 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238401 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758625 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413952 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358584 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589895 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487353 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476168 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129863 s, 404 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619518 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0978198 s, 536 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128735 s, 407 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094869 s, 553 MB/s 52428800 bytes (52 MB) copied, 0.10247 s, 512 MB/s 52428800 bytes (52 MB) copied, 0.101704 s, 516 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100232 s, 523 MB/s 52428800 bytes (52 MB) copied, 0.100808 s, 520 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.107536 s, 488 MB/s 52428800 bytes (52 MB) copied, 0.0800524 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982567 s, 534 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147349 s, 356 MB/s 52428800 bytes (52 MB) copied, 0.213187 s, 246 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817531 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117742 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934738 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125088 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665233 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113635 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128211 s, 409 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933752 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840874 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058283 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.088216 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054137 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033715 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604527 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550196 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280625 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319863 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334443 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328976 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292708 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513049 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037044 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355268 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038169 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348655 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026265 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204324 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183447 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299393 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220091 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310535 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271491 s, 1.9 GB/s , 0.0344809 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263529 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231959 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415365 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291066 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633407 s, 828 MB/s 52428800 bytes (52 MB) copied, 0.081001 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662583 s, 791 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572505 s, 916 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075316 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643145 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553744 s, 947 MB/s 52428800 bytes (52 MB) copied, 0.0614584 s, 853 MB/s 52428800 bytes (52 MB) copied, 0.0780168 s, 672 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243217 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788633 s, 665 MB/s 52428800 bytes (52 MB) copied, 0.0599964 s, 874 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201252 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429481 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272807 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0399146 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338982 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277523 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428361 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301008 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184831 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022099 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174391 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029493 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025199 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264718 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0186454 s, 2.8 GB/s , 0.0276745 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222794 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182254 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310837 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311665 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204112 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254068 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237543 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168891 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184489 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321259 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362314 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343331 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0290526 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303478 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689665 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422666 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381987 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402764 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444705 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712177 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106071 s, 494 MB/s 52428800 bytes (52 MB) copied, 0.0420954 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490404 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343425 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486063 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472964 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380548 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287322 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06376 s, 822 MB/s 52428800 bytes (52 MB) copied, 0.0434541 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216056 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265914 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769995 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400779 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491893 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612249 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418117 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201251 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289523 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180904 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169289 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279593 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175742 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221945 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317927 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302541 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323473 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332839 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348744 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.027905 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035579 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401515 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517441 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478336 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289162 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038469 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253651 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184949 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02132 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283037 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020434 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300368 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294263 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186154 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391629 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382697 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392266 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304562 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618375 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161734 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295804 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049691 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285752 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162502 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402989 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329884 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456024 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296484 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0597238 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310184 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039186 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242323 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695974 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585478 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311881 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413042 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228837 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296484 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352601 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257867 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394445 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227843 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360284 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276331 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310933 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472627 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.030653 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372285 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018376 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313056 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559197 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415813 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594131 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561758 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580838 s, 903 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541044 s, 969 MB/s 52428800 bytes (52 MB) copied, 0.058072 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074558 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381525 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04495 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0935972 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0913958 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958273 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476112 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104555 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546342 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982821 s, 533 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730694 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337869 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0753915 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550425 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395538 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735398 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263465 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627392 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042795 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333708 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222089 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309648 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217662 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379589 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564171 s, 929 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684206 s, 766 MB/s 52428800 bytes (52 MB) copied, 0.0655498 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0951006 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101137 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631455 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549385 s, 954 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117952 s, 444 MB/s 52428800 bytes (52 MB) copied, 0.0627067 s, 836 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124979 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563315 s, 931 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.13562 s, 387 MB/s , 0.0910508 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700864 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886519 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502097 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682133 s, 769 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050209 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0938121 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366103 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547718 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281286 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589835 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269618 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188134 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485148 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185028 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537883 s, 975 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377836 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0307645 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490409 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625439 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045728 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433893 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580132 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716029 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240096 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161942 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286222 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312797 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126555 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448564 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060076 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623731 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276666 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675686 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225642 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03544 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845059 s, 620 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140913 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637981 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689654 s, 760 MB/s 52428800 bytes (52 MB) copied, 0.0514051 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667086 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882793 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132672 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776776 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109992 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276161 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.095954 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108667 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697117 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129341 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920902 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564109 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102333 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172107 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295669 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282871 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154222 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291295 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375784 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377231 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564165 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076455 s, 686 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0922024 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424847 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0363014 s, 1.4 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127179 s, 412 MB/s , 0.0953047 s, 550 MB/s 52428800 bytes (52 MB) copied, 0.0635914 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0955231 s, 549 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163969 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822538 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156785 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0747336 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969126 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298713 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025572 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175953 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541432 s, 968 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574066 s, 913 MB/s 52428800 bytes (52 MB) copied, 0.0716722 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0934811 s, 561 MB/s 52428800 bytes (52 MB) copied, 0.0403955 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541612 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326387 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635916 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0887501 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904441 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388445 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069578 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027677 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421548 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664954 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946846 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102964 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304554 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639577 s, 820 MB/s 52428800 bytes (52 MB) copied, 0.0730688 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500934 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797795 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300153 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166525 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267614 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365865 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033985 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371742 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366718 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318462 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284402 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271277 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197722 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162982 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364916 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.030627 s, 1.7 GB/s , 0.0406047 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605651 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402705 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448169 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265877 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333048 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215545 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329661 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155931 s, 3.4 GB/s 52428800 bytes (52 MB) copied, 0.0468521 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365138 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313113 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353028 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441087 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366356 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0841425 s, 623 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419846 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580871 s, 903 MB/s 52428800 bytes (52 MB) copied, 0.0220731 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225054 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247967 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0484375 s, 1.1 GB/s , 0.0463587 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212901 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341712 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236734 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352873 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307767 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241316 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248221 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238419 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0895597 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791186 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941291 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581994 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0856161 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732967 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109987 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134304 s, 390 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119395 s, 439 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101797 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101624 s, 516 MB/s 52428800 bytes (52 MB) copied, 0.146425 s, 358 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109466 s, 479 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.096479 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672646 s, 779 MB/s 52428800 bytes (52 MB) copied, 0.0959424 s, 546 MB/s 52428800 bytes (52 MB) copied, 0.0611452 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934782 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869108 s, 603 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165175 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831011 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548383 s, 956 MB/s 52428800 bytes (52 MB) copied, 0.123818 s, 423 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132591 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.145163 s, 361 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031942 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050576 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660621 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731532 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031131 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283868 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388877 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318239 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335307 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0164236 s, 3.2 GB/s , 0.0268638 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551475 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337965 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267967 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.051839 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0194409 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437485 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369507 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267185 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235614 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042642 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574802 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477377 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514258 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310233 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483054 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493472 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402682 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446985 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425516 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287975 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041215 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151072 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122168 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423467 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069901 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312214 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276274 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122637 s, 428 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123228 s, 425 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0944026 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092003 s, 570 MB/s 52428800 bytes (52 MB) copied, 0.147838 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855592 s, 613 MB/s 52428800 bytes (52 MB) copied, 0.12008 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126891 s, 413 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920742 s, 569 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507499 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.130155 s, 403 MB/s 52428800 bytes (52 MB) copied, 0.177999 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667098 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110959 s, 473 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811629 s, 646 MB/s 52428800 bytes (52 MB) copied, 0.0780619 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286979 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123032 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115801 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416826 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752803 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251819 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418455 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299706 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177186 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256692 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472028 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241352 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0557331 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468318 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156865 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402191 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487486 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596297 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298349 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046454 s, 1.1 GB/s , 0.0402169 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077954 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102517 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518991 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646926 s, 810 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697093 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644424 s, 814 MB/s 52428800 bytes (52 MB) copied, 0.0855081 s, 613 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106707 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065974 s, 795 MB/s 52428800 bytes (52 MB) copied, 0.0740461 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13965 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130955 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644442 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726854 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664202 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117548 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10749 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732876 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671293 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549287 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0905706 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316472 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231607 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470415 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124345 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370467 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214166 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0930796 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234583 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288681 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016364 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282038 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173497 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205734 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212931 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154397 s, 3.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315601 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0338476 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347862 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500792 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419173 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425975 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0328402 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188537 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780677 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492375 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056286 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680474 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869381 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10473 s, 501 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10233 s, 512 MB/s 52428800 bytes (52 MB) copied, 0.13493 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0984134 s, 533 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106845 s, 491 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117201 s, 447 MB/s 52428800 bytes (52 MB) copied, 0.0735008 s, 713 MB/s , 0.121652 s, 431 MB/s 52428800 bytes (52 MB) copied, 0.0935342 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753208 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122138 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125903 s, 416 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.105451 s, 497 MB/s , 0.0981762 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106053 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611223 s, 858 MB/s 52428800 bytes (52 MB) copied, 0.0735345 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105697 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493861 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525997 s, 997 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566684 s, 925 MB/s 52428800 bytes (52 MB) copied, 0.0838644 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481168 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330518 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637763 s, 822 MB/s 52428800 bytes (52 MB) copied, 0.0477025 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594106 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028412 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331467 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036247 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165412 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184034 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335617 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264833 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170487 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268663 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229027 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261618 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354271 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247885 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203623 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299771 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339391 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273228 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357919 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223807 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210767 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313756 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558626 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449351 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393726 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436452 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388839 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372516 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0530614 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100921 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487273 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337518 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154039 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414651 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266207 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530888 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524574 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325663 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024399 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582482 s, 900 MB/s 52428800 bytes (52 MB) copied, 0.066755 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651763 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657661 s, 797 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101255 s, 518 MB/s 52428800 bytes (52 MB) copied, 0.0687562 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115188 s, 455 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542758 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11511 s, 455 MB/s 52428800 bytes (52 MB) copied, 0.1418 s, 370 MB/s 52428800 bytes (52 MB) copied, 0.127381 s, 412 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0902192 s, 581 MB/s 52428800 bytes (52 MB) copied, 0.0645669 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111662 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592674 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746168 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131431 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185623 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404993 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103161 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114483 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.197982 s, 265 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15535 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069822 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566993 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107566 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339139 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036685 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392567 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452752 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216066 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301705 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275312 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159343 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043632 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020168 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237372 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194186 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363264 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535539 s, 979 MB/s 52428800 bytes (52 MB) copied, 0.0498537 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649818 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09417 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722178 s, 726 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821941 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042035 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114852 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871463 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548392 s, 956 MB/s 52428800 bytes (52 MB) copied, 0.106013 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565357 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114623 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619404 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547097 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109237 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981581 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971922 s, 539 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622356 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460209 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0433529 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030451 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853824 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060086 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669823 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358451 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589934 s, 889 MB/s 52428800 bytes (52 MB) copied, 0.0975162 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427964 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635817 s, 825 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337334 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0552823 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072067 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045304 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574325 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417163 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767602 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812727 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949182 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517699 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264999 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028783 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187616 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.014494 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302462 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249926 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252108 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232549 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285732 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246562 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383086 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223854 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369533 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226991 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237908 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279792 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351922 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248309 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197965 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318012 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237899 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207902 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409571 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0323914 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308458 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0633281 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287595 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02634 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216414 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337929 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640549 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673164 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237091 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371115 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512609 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462067 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614054 s, 854 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404731 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0479587 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621334 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428339 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064189 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453235 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0244339 s, 2.1 GB/s , 0.0399157 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525401 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090623 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238283 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166985 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263566 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435101 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550348 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609182 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594776 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566261 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567931 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361542 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026041 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505165 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309617 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0561219 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032924 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295453 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365676 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171031 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201772 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373733 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234129 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190855 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180083 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150363 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495963 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226071 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262329 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159443 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03181 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397556 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279532 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029325 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412062 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.040225 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754578 s, 695 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080048 s, 655 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101113 s, 519 MB/s 52428800 bytes (52 MB) copied, 0.09904 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0987869 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782805 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126363 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093053 s, 563 MB/s 52428800 bytes (52 MB) copied, 0.0806538 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777249 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0926399 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824314 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035697 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08428 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728821 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964047 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566193 s, 926 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664321 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249635 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0769867 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295365 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431146 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529064 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289314 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0489486 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185743 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341179 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566795 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264401 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358641 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0522619 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220322 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316466 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336948 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312104 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256649 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195317 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185938 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282888 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368376 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210391 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187731 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038361 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367467 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342409 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225869 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315512 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381443 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0444083 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510382 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033923 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934383 s, 561 MB/s 52428800 bytes (52 MB) copied, 0.0861603 s, 609 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109906 s, 477 MB/s 52428800 bytes (52 MB) copied, 0.0591099 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715621 s, 733 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112141 s, 468 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128109 s, 409 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516679 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0964046 s, 544 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0739527 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838237 s, 625 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890012 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919162 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074703 s, 702 MB/s , 0.0656649 s, 798 MB/s 52428800 bytes (52 MB) copied, 0.0863571 s, 607 MB/s , 0.0728244 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496483 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074957 s, 699 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016706 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281562 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409934 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.125733 s, 417 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170516 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0356556 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961275 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345787 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164738 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246413 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245361 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194597 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393181 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205867 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451352 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292089 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367113 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298619 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345738 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190612 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276423 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672192 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242704 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435929 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752598 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296549 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512893 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030574 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0457164 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435228 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522089 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847257 s, 619 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.108304 s, 484 MB/s , 0.121216 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066901 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796099 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777579 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11623 s, 451 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800128 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103779 s, 505 MB/s 52428800 bytes (52 MB) copied, 0.0735771 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055836 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104943 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961595 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120456 s, 435 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16826 s, 312 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.104464 s, 502 MB/s 52428800 bytes (52 MB) copied, 0.109761 s, 478 MB/s 52428800 bytes (52 MB) copied, 0.0942298 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055343 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090752 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04835 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281841 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239144 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528375 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239894 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229625 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271505 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239564 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238413 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021526 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244623 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348519 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418219 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522839 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623934 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403281 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674178 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381553 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591849 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361527 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465986 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0427266 s, 1.2 GB/s , 0.0384901 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389969 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290994 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690818 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271694 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345378 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0743763 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917831 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494125 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234074 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538771 s, 973 MB/s 52428800 bytes (52 MB) copied, 0.0379696 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0641729 s, 817 MB/s , 0.083133 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777028 s, 675 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0773147 s, 678 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0684207 s, 766 MB/s 52428800 bytes (52 MB) copied, 0.0493534 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18572 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108671 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316268 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847269 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077967 s, 672 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146668 s, 357 MB/s 52428800 bytes (52 MB) copied, 0.0672195 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479421 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0328825 s, 1.6 GB/s , 0.0515362 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11646 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735911 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0927686 s, 565 MB/s 52428800 bytes (52 MB) copied, 0.118751 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689429 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421283 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184056 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288285 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365989 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323888 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348675 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395882 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051476 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139002 s, 377 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703215 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660283 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069663 s, 753 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678414 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665719 s, 788 MB/s 52428800 bytes (52 MB) copied, 0.0624705 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552123 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691109 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117375 s, 447 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068326 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705854 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124618 s, 421 MB/s /mnt/lustre/f45.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x1a5:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800a8a06800.stats= snapshot_time 1746132488.987156523 secs.nsecs start_time 1746132428.401077111 secs.nsecs elapsed_time 60.586079412 secs.nsecs read_bytes 1551 samples [bytes] 135168 1048576 1625292800 1703996095987712 read 150000 samples [usecs] 137 226770 118722022 1501237439370 ioctl 1 samples [reqs] open 3001 samples [usecs] 4 17402 174438 1241727582 close 3001 samples [usecs] 5 34180 370614 3977079138 seek 2999 samples [usecs] 2 1324 15103 1996841 setattr 1 samples [usecs] 11700 11700 11700 136890000 getattr 32 samples [usecs] 115 1616 7095 3834907 mknod 1 samples [usecs] 15138 15138 15138 229159044 inode_permission 12060 samples [usecs] 1 16927 1084010 1617781086 opencount 3002 samples [reqs] 1 3001 4504502 9013506502 openclosetime 3000 samples [usecs] 2 127886 54912796 2022265967828 pcc_attach 1 samples [reqs] pcc_hit_bytes 2969 samples [bytes] 52428800 52428800 155661107200 8161125057167360000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff8800b394c800.stats= snapshot_time 1746132488.987289172 secs.nsecs start_time 1746132428.401096874 secs.nsecs elapsed_time 60.586192298 secs.nsecs read_bytes 1150 samples [bytes] 1048576 1048576 1205862400 1264438371942400 read 151350 samples [usecs] 138 161239 125853997 1809396992411 open 3027 samples [usecs] 3 5099 87243 161015555 close 3027 samples [usecs] 6 27842 243715 1757007307 seek 3026 samples [usecs] 2 36820 56562 1366958768 inode_permission 6052 samples [usecs] 1 16154 410660 959301574 opencount 3027 samples [reqs] 1 3011 4534582 9103897022 openclosetime 3010 samples [usecs] 2 156644 54918070 2161761313280 pcc_attach 1 samples [reqs] pcc_hit_bytes 3004 samples [bytes] 52428800 52428800 157496115200 8257332324597760000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 attach 2 detach 0 auto_attach 0 llite.lustre-ffff8800a8a06800.pcc_async_threshold=1073741824 llite.lustre-ffff8800b394c800.pcc_async_threshold=1073741824 /mnt/pcc.d45.sanity-pcc.2 is idle now /mnt/pcc.d45.sanity-pcc is idle now PASS 45 (67s) == sanity-pcc test 46: Verify PCC mode setting works correctly ========================================================== 16:48:13 (1746132493) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0890433 s, 707 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d46.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: f34f23e8-a9ef-4ee6-9ae5-a0aa7cd7dbfd Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f46.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f34f23e8-a9ef-4ee6-9ae5-a0aa7cd7dbfd (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d46.sanity-pcc/d46.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} running as uid/gid/euid/egid 500/500/500/500, groups: 0 [id] uid=500(sanityusr) gid=500(sanityusr) groups=500(sanityusr),0(root) Mode: 0400 llite.lustre-ffff8800a8a06800.pcc_mode=0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800a8a06800.pcc_mode=0400 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800a8a06800.pcc_mode=0004 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [stat] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 1 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0664/-rw-rw-r--) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-01 16:48:15.000000000 -0400 Modify: 2025-05-01 16:48:14.000000000 -0400 Change: 2025-05-01 16:48:15.000000000 -0400 Birth: - running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800a8a06800.pcc_mode=0040 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 4 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0660/-rw-rw----) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-01 16:48:15.000000000 -0400 Modify: 2025-05-01 16:48:14.000000000 -0400 Change: 2025-05-01 16:48:15.000000000 -0400 Birth: - file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800a8a06800.pcc_mode=0400 llite.lustre-ffff8800a8a06800.pcc_mode=0400 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d46.sanity-pcc is idle now PASS 46 (5s) == sanity-pcc test 47: mtime should be kept once file attached into PCC ========================================================== 16:48:18 (1746132498) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0813503 s, 773 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d47.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: c2c3fb92-707a-46fe-8dc6-9fc783d79f2c Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f47.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c2c3fb92-707a-46fe-8dc6-9fc783d79f2c (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d47.sanity-pcc/d47.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} QQQQQ /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/f47.sanity-pcc | grep -E -c 'type: readonly' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d47.sanity-pcc is idle now PASS 47 (7s) == sanity-pcc test 48: PCC state should check whether the file in local PCC cache ========================================================== 16:48:25 (1746132505) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0878774 s, 716 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d48.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 39da6892-da09-4e62-b220-84f971b40a8d Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f48.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=39da6892-da09-4e62-b220-84f971b40a8d (extents) (64bit) (large files) (huge files) QQQQQ file: /mnt/lustre/f48.sanity-pcc, type: readonly, PCC_file: /d48.sanity-pcc/59fd/0x2000059f1:0xc:0x0 (unlinked), open_count: 0, flags: 4 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d48.sanity-pcc is idle now PASS 48 (4s) == sanity-pcc test 96: Auto attach from multiple read process on a node ========================================================== 16:48:29 (1746132509) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0814932 s, 772 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d96.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 57089623-0dbb-44b8-8265-492eccd041ac Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f96.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=57089623-0dbb-44b8-8265-492eccd041ac (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d96.sanity-pcc/d96.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8a06800.pcc_async_threshold=1G llite.lustre-ffff8800b394c800.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.617596 s, 84.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.647126 s, 81.0 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.675745 s, 77.6 MB/s , 0.703597 s, 74.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.663483 s, 79.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.695896 s, 75.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455935 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307286 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189973 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351007 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197963 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240785 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175288 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405581 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244589 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163775 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0352794 s, 1.5 GB/s , 0.0227776 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202806 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398009 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227655 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180151 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240461 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188175 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185197 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246611 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149322 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271193 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186756 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391086 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308587 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180484 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335041 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164734 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162922 s, 3.2 GB/s , 0.0153136 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223954 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434941 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474072 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616019 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059385 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367641 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258622 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425721 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269103 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181918 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026442 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225361 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184708 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444811 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396916 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189215 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029614 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180821 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250726 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181739 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197249 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371302 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222392 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234143 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198004 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231662 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200476 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179156 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172593 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169808 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284356 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020508 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336401 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263694 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377589 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284483 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262613 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198341 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227798 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165866 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171247 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038822 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202571 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276579 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238766 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0473995 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214092 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204297 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192254 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162012 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265254 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196605 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216636 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213973 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196245 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193915 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352405 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177704 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305726 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253329 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218036 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342846 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236567 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186385 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321557 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015545 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172765 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360774 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228589 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303257 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224086 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156357 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219015 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190108 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340063 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160387 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157106 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209954 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369042 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438114 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028642 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238869 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237004 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175428 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230309 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281846 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302924 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263319 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02211 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283206 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260072 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209613 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226981 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332548 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271167 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190889 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192075 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204342 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151164 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01873 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216832 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043545 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029371 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417021 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210558 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022489 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348891 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017567 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214217 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021722 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299613 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389006 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302839 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150466 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643757 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292171 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018967 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234839 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223865 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315263 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207956 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349724 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217582 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198813 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201397 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384034 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244197 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193191 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290144 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353386 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222076 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521558 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290487 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245271 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578357 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245776 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275605 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173742 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277375 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282189 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290629 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330736 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241656 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219662 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312222 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161637 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218753 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198451 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246881 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161984 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027108 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262146 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238078 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196693 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201534 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556289 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488163 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277074 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254891 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273095 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251944 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279261 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550257 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320687 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232529 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202565 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375591 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354416 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250558 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229199 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263525 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345614 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221094 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229398 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324103 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180445 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029139 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179758 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203424 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221722 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021046 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244942 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282123 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367862 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187621 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396133 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291098 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331864 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224085 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178236 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311723 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0380819 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295065 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325899 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208396 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223813 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226629 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204826 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226853 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250939 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210184 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254825 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324835 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609034 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346349 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194819 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230466 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244888 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245505 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240253 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335598 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488329 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585355 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342146 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512517 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303925 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520052 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155445 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040022 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202457 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505255 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250682 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238854 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340807 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149943 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409608 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265037 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157865 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023694 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021699 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250603 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254606 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274415 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252672 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029754 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239079 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258309 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176584 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223853 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204826 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273393 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259735 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298962 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168509 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322901 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173027 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213582 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174215 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204317 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333587 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218654 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364744 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336756 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355322 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156032 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511302 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333599 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264849 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258955 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310136 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229625 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300969 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440023 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582976 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248477 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406628 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345133 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203896 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217484 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250031 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169245 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279459 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200449 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273957 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179673 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179026 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022859 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276559 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304247 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302966 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300785 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330312 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233507 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044381 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243611 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238785 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189854 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271694 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351396 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215675 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208097 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506156 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325414 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327125 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371198 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239417 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260222 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030199 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230138 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197124 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021703 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272978 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314108 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176371 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282577 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200645 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250328 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347054 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197926 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541623 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165955 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604478 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706027 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480434 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486099 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021592 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334487 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046559 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211747 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027562 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278073 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308699 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156682 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539514 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622047 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797302 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383147 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279265 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459702 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179049 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172782 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268403 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492899 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256514 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514657 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251744 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516234 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542323 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384239 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285194 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304523 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359058 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407626 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510746 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249104 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327927 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334673 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406411 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519978 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304657 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309301 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228506 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295102 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194304 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150597 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209074 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363433 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312938 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594789 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378294 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208925 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275594 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244135 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346468 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165578 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371418 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172358 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182495 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211098 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252236 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211854 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178982 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417535 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164516 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357597 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410231 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259302 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202007 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594469 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064468 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023906 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287826 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223765 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241402 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202809 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216191 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199677 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266491 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329625 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265771 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443537 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254667 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228872 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212594 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018509 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269472 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327124 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379653 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295068 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269736 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209551 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178033 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196968 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195404 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206597 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179112 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208733 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310903 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303343 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453388 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231021 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341205 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404226 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204772 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025715 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229275 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338514 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191656 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262424 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498067 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491718 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341876 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494229 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229212 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620932 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301176 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521343 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465837 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298664 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599571 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838121 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341406 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355602 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339103 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450588 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531141 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429475 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504819 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534139 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150599 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619677 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445989 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562346 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487047 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017887 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166196 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477565 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274248 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222267 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619203 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695293 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249062 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255417 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334129 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588714 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388667 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184393 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388123 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417863 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023926 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033719 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489772 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481077 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301178 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235145 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231619 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220243 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239698 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024067 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553617 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391277 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043986 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434264 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237807 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185564 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219923 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209319 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251748 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671206 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857034 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608723 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732783 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561006 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282873 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203659 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286846 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231106 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192851 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163519 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191008 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177878 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346839 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059477 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059781 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610176 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211626 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258875 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265883 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407631 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368962 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185409 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230603 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029899 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427331 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292636 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504032 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354923 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027632 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551208 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346008 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251045 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195052 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240635 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239811 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287656 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040198 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217323 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324088 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278919 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569251 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546502 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723903 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223942 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447529 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344064 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282624 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260252 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164335 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286947 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165863 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455152 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202825 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213469 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210512 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190224 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261118 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030324 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279208 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365622 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514633 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435975 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307248 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227252 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331581 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296255 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153677 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353179 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275026 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214402 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303577 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252509 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156489 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185603 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149113 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218745 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340216 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188396 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169796 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226502 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458205 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204506 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164331 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372298 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257605 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271969 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156915 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216213 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308511 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165607 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234263 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027637 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345899 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357513 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306167 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195315 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192814 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386297 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262485 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191767 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173726 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448424 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508955 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287598 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023144 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293949 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458336 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442318 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242637 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301153 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273487 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311977 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273315 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171892 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329644 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565166 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478528 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199689 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275413 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180472 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016239 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212224 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178376 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226674 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283217 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265038 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344893 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364073 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225939 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166846 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197799 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022653 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548828 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290218 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317097 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366389 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398345 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270944 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340002 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028353 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227878 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02203 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286351 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318423 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200255 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229281 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148156 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182588 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460304 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447993 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186854 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339786 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256975 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536965 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015062 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356684 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042645 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372985 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206101 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393902 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297821 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458466 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043791 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282834 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218668 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268452 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235247 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461967 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401072 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258942 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301551 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155685 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163564 s, 3.2 GB/s , 0.050226 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208973 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229865 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195128 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382503 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186457 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419832 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464117 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033294 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347763 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252151 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382538 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269454 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439115 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218348 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623017 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269602 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215728 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022176 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205565 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224464 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329442 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334554 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576162 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203896 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201452 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251301 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387395 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299538 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225395 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025103 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0331283 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370015 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295526 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447502 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408394 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052132 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207948 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361301 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452713 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612927 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364463 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538376 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321665 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294729 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402119 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358702 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152659 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504386 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044703 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482592 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460105 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339274 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374874 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371151 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338298 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201666 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278539 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356059 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040064 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343219 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355997 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415476 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298223 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461558 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304562 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448378 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466678 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286525 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030127 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510596 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250273 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337473 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511858 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248175 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237924 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178763 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407458 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529042 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308678 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303614 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367528 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411948 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276354 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477746 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163935 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251945 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029701 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229055 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557339 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634574 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581383 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705241 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053219 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561903 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264385 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357386 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424747 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633432 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813497 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318196 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381171 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213051 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031567 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323453 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372903 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359343 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248707 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374844 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153166 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261002 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826389 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053348 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155216 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414921 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433705 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675552 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570637 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403928 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439671 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210346 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262135 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188809 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248585 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024721 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450771 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536172 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597229 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653211 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558256 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306055 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387675 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556607 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056432 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746745 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436424 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416398 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865358 s, 606 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350509 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225606 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048602 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432824 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534374 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563847 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640769 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022677 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288063 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347605 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484006 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228495 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551395 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403383 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316686 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285761 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275228 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162389 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327623 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157617 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160532 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513005 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263509 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315365 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252909 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335937 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246236 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510108 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241176 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266841 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016477 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435866 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330769 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328232 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264765 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391422 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282995 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178595 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327781 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331672 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173509 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214303 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316021 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229159 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322107 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665481 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532656 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242702 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275971 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460375 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401996 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239425 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339659 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167055 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025855 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313522 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036612 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043934 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265021 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235371 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334454 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465947 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299908 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641016 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219947 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648448 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301529 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478224 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315231 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353227 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245006 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189343 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284712 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471879 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399022 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726969 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475543 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464552 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431602 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536548 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020856 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024441 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307539 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403364 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0145819 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223091 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511358 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251872 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221263 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179147 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363306 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295186 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174877 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245662 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199175 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392982 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532272 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420843 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039046 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682602 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556152 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310856 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207271 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394695 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241856 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348066 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185503 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053974 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274148 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187675 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431333 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655867 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282467 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309028 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376472 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026544 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290397 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158285 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355858 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377327 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263922 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275092 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018336 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251143 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043706 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036516 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579509 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272753 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272752 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308242 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342044 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271314 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228017 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261229 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579549 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269142 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178762 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381381 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332715 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155755 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039159 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286975 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277049 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359145 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468979 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664351 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255304 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272748 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249218 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387787 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502046 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387413 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345998 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180297 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425025 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527781 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305297 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401504 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385666 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453199 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538003 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403446 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468486 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508062 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121566 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305325 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362708 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064759 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464964 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184941 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260442 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350778 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239348 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296187 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231378 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357524 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357463 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369088 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562549 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578903 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273912 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028103 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495709 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301048 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034809 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250189 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147552 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324877 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188425 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024732 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371136 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318973 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447325 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235292 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251247 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294185 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245962 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207202 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016984 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276979 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258716 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401122 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246909 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315486 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356732 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217915 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470686 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426726 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238347 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247039 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376228 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029428 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227334 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283488 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234413 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271183 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031864 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212548 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277466 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173772 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03029 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196526 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168875 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253257 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330228 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279198 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263797 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341937 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152635 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203425 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413159 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568785 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822334 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976314 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992346 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525096 s, 998 MB/s Finish ======== detach: must specify one or more FIDs Detach given files from the Persistent Client Cache. usage: lfs pcc detach [-k] FILE [...] lfs commands used to interact with PCC features: lfs pcc attach - attach given files to Persistent Client Cache lfs pcc attach_fid - attach given files into PCC by FID(s) lfs pcc state - display the PCC state for given files lfs pcc detach - detach given files from Persistent Client Cache lfs pcc detach_fid - detach given files from PCC by FID(s) lfs pcc delete - delete the PCC layout componenet for given files lfs pcc pin - pin give files for PCC lfs pcc unpin - unpin given files for PCC llite.lustre-ffff8800a8a06800.pcc_async_threshold=1073741824 llite.lustre-ffff8800b394c800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d96.sanity-pcc is idle now PASS 96 (66s) == sanity-pcc test 97: two mmap I/O and layout lock cancel ========================================================== 16:49:35 (1746132575) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.110614 s, 569 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d97.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 2fbc68bb-bbf5-4565-a9f8-edc86ace84fa Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f97.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2fbc68bb-bbf5-4565-a9f8-edc86ace84fa (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d97.sanity-pcc/d97.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8a06800.pcc_async_threshold=1G llite.lustre-ffff8800b394c800.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.663543 s, 79.0 MB/s T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... llite.lustre-ffff8800a8a06800.pcc_async_threshold=1073741824 llite.lustre-ffff8800b394c800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d97.sanity-pcc is idle now PASS 97 (128s) == sanity-pcc test 98: racer between auto attach and mmap I/O ========================================================== 16:51:43 (1746132703) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0737733 s, 853 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d98.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 0915e7d0-75bc-4df4-bf31-67480feef74f Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f98.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0915e7d0-75bc-4df4-bf31-67480feef74f (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d98.sanity-pcc/d98.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8a06800.pcc_async_threshold=0 llite.lustre-ffff8800b394c800.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.44881 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.571074 s, 91.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.58913 s, 89.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.609609 s, 86.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242973 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381247 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020281 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189175 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677216 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155191 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768563 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349519 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332382 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373932 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259227 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027575 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302326 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337174 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204622 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170311 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333525 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390269 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03005 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463559 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.30287 s, 40.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.30424 s, 40.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.41085 s, 37.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18019 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102551 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.297631 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.605909 s, 86.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278184 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273951 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.331362 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.343119 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0733456 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782051 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.526459 s, 99.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.685492 s, 76.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.484103 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.306648 s, 171 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.649269 s, 80.8 MB/s 52428800 bytes (52 MB) copied, 0.651658 s, 80.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17269 s, 304 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174195 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.390033 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147732 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154493 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.347851 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.444263 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.374503 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246011 s, 213 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352394 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259206 s, 202 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.482536 s, 109 MB/s , 0.182027 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.42701 s, 123 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.141137 s, 371 MB/s 52428800 bytes (52 MB) copied, 0.0539654 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150869 s, 348 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.811115 s, 64.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.818089 s, 64.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.576274 s, 91.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120137 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226184 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269358 s, 195 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.515103 s, 102 MB/s , 0.356021 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.308305 s, 170 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.299833 s, 175 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519515 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.71909 s, 72.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253938 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439855 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033061 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195254 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275151 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271477 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589457 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280872 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329027 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560914 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424676 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432087 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533183 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275466 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215737 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331813 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297494 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299771 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270563 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345264 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444699 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553197 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388938 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252139 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457294 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382266 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0397937 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329164 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036629 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332466 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569951 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341911 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464747 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840107 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023946 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289006 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.879082 s, 59.6 MB/s , 0.947098 s, 55.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.879657 s, 59.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.506387 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443218 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.620178 s, 84.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.658054 s, 79.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.483501 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.605457 s, 86.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.84436 s, 62.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.64367 s, 81.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.730657 s, 71.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.584338 s, 89.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.69207 s, 75.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.84486 s, 62.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.503805 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.353662 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.488045 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443666 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.379734 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.671355 s, 78.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.66524 s, 78.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.673903 s, 77.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.625749 s, 83.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112761 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917398 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149349 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.18124 s, 44.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.18454 s, 44.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.19758 s, 43.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.324289 s, 162 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.336529 s, 156 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.448879 s, 117 MB/s , 0.202777 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135764 s, 386 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.574205 s, 91.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903926 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.549153 s, 95.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.906743 s, 57.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.82971 s, 63.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264518 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.301611 s, 174 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.488165 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.414295 s, 127 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.535709 s, 97.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.541526 s, 96.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.502635 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.439807 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204943 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.447709 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.771854 s, 67.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.773011 s, 67.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.876085 s, 59.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183626 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195029 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.416023 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.419917 s, 125 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.654514 s, 80.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.433347 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.199185 s, 263 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.762516 s, 68.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.36831 s, 142 MB/s 52428800 bytes (52 MB) copied, 0.367662 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.474137 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198262 s, 264 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694507 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116504 s, 450 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.52417 s, 100 MB/s 52428800 bytes (52 MB) copied, 0.522532 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.469582 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.115656 s, 453 MB/s 52428800 bytes (52 MB) copied, 0.45049 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116842 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359132 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.467011 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155724 s, 337 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.379654 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.615442 s, 85.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.960875 s, 54.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.970027 s, 54.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121902 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239942 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.887673 s, 59.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.709013 s, 73.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.653797 s, 80.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.202063 s, 259 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267594 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.487211 s, 108 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.155146 s, 338 MB/s , 0.156688 s, 335 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022568 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.713527 s, 73.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202824 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316218 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234065 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345954 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221029 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187517 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450928 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0257666 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0394013 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340175 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341729 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182224 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208051 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238104 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317346 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552233 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250803 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433809 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059203 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364287 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316005 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035156 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221777 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233851 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248488 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017537 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407939 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451779 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370636 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537711 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270278 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.769631 s, 68.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.761263 s, 68.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.547726 s, 95.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780425 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.313026 s, 167 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318002 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.322843 s, 162 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.391276 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.71838 s, 73.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.716784 s, 73.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573822 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236717 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206992 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245776 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336468 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191909 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428057 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461235 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414386 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201734 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635775 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555562 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248304 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262391 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435494 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303201 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347507 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215171 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187669 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203263 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315188 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027879 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171494 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276508 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203224 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245627 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186335 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233098 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405491 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023484 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218702 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407115 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208004 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323707 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203505 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024436 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334936 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.291939 s, 180 MB/s , 0.506076 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.506815 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.358465 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.360764 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.366211 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130513 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285017 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360107 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.84414 s, 62.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.570962 s, 91.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.782663 s, 67.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.536917 s, 97.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.535548 s, 97.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489181 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415312 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.344069 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.417167 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.537463 s, 97.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.832785 s, 63.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.932061 s, 56.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260933 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142735 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335353 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339272 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495094 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482295 s, 1.1 GB/s llite.lustre-ffff8800a8a06800.pcc_async_threshold=1073741824 llite.lustre-ffff8800b394c800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d98.sanity-pcc is idle now PASS 98 (66s) == sanity-pcc test 99: race among unlink | mmap read | write | detach for PCC-RO file ========================================================== 16:52:49 (1746132769) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.086376 s, 728 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: e5274441-3867-431b-b7ac-4170ebb198c0 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f99.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e5274441-3867-431b-b7ac-4170ebb198c0 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.473614 s, 111 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.694506 s, 75.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187115 s, 2.8 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.358496 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.455189 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325153 s, 1.6 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.453937 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672754 s, 77.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.786701 s, 66.6 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242762 s, 2.2 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.779568 s, 67.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.41221 s, 127 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.740639 s, 70.8 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.78755 s, 66.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.661982 s, 79.2 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.668256 s, 78.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352507 s, 1.5 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.87365 s, 60.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177269 s, 3.0 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.723713 s, 72.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.795151 s, 65.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.688415 s, 76.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.580962 s, 90.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.485315 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.366109 s, 143 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296797 s, 1.8 GB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.361949 s, 145 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321314 s, 1.6 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.411723 s, 127 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.677437 s, 77.4 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.533549 s, 98.3 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.327199 s, 160 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.231313 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.502287 s, 104 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.593808 s, 88.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231824 s, 2.3 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.7229 s, 72.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.67417 s, 77.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.601113 s, 87.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.722586 s, 72.6 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.623475 s, 84.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.441874 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.576362 s, 91.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457635 s, 115 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.851251 s, 61.6 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318312 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.363748 s, 144 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318157 s, 165 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.979914 s, 53.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.296751 s, 177 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33984 s, 154 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.402164 s, 130 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.394275 s, 133 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.670095 s, 78.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.681229 s, 77.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226694 s, 2.3 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.654956 s, 80.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.527984 s, 99.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.647016 s, 81.0 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.857524 s, 61.1 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0968341 s, 541 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.701797 s, 74.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425897 s, 1.2 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445664 s, 118 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.732163 s, 71.6 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.625459 s, 83.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193779 s, 2.7 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.582045 s, 90.1 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.673014 s, 77.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.743462 s, 70.5 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.476611 s, 110 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.489835 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.412424 s, 127 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.281369 s, 186 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.595984 s, 88.0 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672309 s, 78.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.79577 s, 65.9 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457509 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514058 s, 1.0 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239645 s, 219 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.59141 s, 88.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.660248 s, 79.4 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.747341 s, 70.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.564367 s, 92.9 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.638659 s, 82.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273503 s, 1.9 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443261 s, 118 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.818144 s, 64.1 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433893 s, 1.2 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.660926 s, 79.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.408471 s, 128 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.756408 s, 69.3 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.41874 s, 125 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155027 s, 338 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.57685 s, 90.9 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.564216 s, 92.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201252 s, 2.6 GB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.04741 s, 50.1 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182943 s, 2.9 GB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.519004 s, 101 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.17545 s, 44.6 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.45531 s, 115 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.613612 s, 85.4 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.69442 s, 75.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.584428 s, 89.7 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.593596 s, 88.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.822716 s, 63.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.621342 s, 84.4 MB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800a8a06800.rpc_stats= snapshot_time: 1746132832.596848107 secs.nsecs start_time: 1746132292.570632977 secs.nsecs elapsed_time: 540.026215130 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1015 6 6 | 3 0 0 2: 12 0 6 | 0 0 0 4: 9 0 6 | 0 0 0 8: 14 0 6 | 0 0 0 16: 3507 21 27 | 0 0 0 32: 361 2 29 | 0 0 0 64: 564 3 32 | 0 0 0 128: 1097 6 39 | 0 0 0 256: 10121 60 100 | 3000 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 9561 57 57 | 231 7 7 2: 1658 9 67 | 493 16 24 3: 779 4 71 | 692 23 47 4: 649 3 75 | 540 17 65 5: 607 3 79 | 427 14 79 6: 475 2 82 | 251 8 87 7: 456 2 84 | 140 4 92 8: 425 2 87 | 101 3 95 9: 1960 11 99 | 121 4 99 10: 15 0 99 | 7 0 100 11: 10 0 99 | 0 0 100 12: 10 0 99 | 0 0 100 13: 9 0 99 | 0 0 100 14: 8 0 99 | 0 0 100 15: 7 0 99 | 0 0 100 16: 6 0 99 | 0 0 100 17: 5 0 99 | 0 0 100 18: 5 0 99 | 0 0 100 19: 4 0 99 | 0 0 100 20: 4 0 99 | 0 0 100 21: 4 0 99 | 0 0 100 22: 4 0 99 | 0 0 100 23: 3 0 99 | 0 0 100 24: 3 0 99 | 0 0 100 25: 3 0 99 | 0 0 100 26: 2 0 99 | 0 0 100 27: 2 0 99 | 0 0 100 28: 2 0 99 | 0 0 100 29: 2 0 99 | 0 0 100 30: 2 0 99 | 0 0 100 31: 20 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 200 1 1 | 63 2 2 1: 77 0 1 | 0 0 2 2: 148 0 2 | 0 0 2 4: 10 0 2 | 0 0 2 8: 4 0 2 | 0 0 2 16: 141 0 3 | 0 0 2 32: 249 1 4 | 0 0 2 64: 425 2 7 | 0 0 2 128: 786 4 12 | 0 0 2 256: 339 2 14 | 60 1 4 512: 671 4 18 | 120 3 8 1024: 1263 7 25 | 240 7 16 2048: 2413 14 40 | 480 15 32 4096: 4859 29 69 | 960 31 64 8192: 5115 30 100 | 1080 35 100 osc.lustre-OST0000-osc-ffff8800b394c800.rpc_stats= snapshot_time: 1746132832.596980266 secs.nsecs start_time: 1746132294.219845666 secs.nsecs elapsed_time: 538.377134600 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4 8 8 | 0 0 0 2: 2 4 12 | 0 0 0 3: 1 2 14 | 0 0 0 4: 1 2 16 | 0 0 0 5: 1 2 18 | 0 0 0 6: 1 2 20 | 0 0 0 7: 1 2 22 | 0 0 0 8: 1 2 24 | 0 0 0 9: 38 76 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800a8a06800.rpc_stats= snapshot_time: 1746132832.597050371 secs.nsecs start_time: 1746132292.571201345 secs.nsecs elapsed_time: 540.025849026 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 0 0 0 | 0 0 1 32: 0 0 0 | 0 0 1 64: 0 0 0 | 0 0 1 128: 0 0 0 | 0 0 1 256: 352 99 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 36 10 10 | 8 5 5 2: 25 7 17 | 55 35 41 3: 18 5 22 | 50 32 73 4: 18 5 27 | 27 17 91 5: 13 3 30 | 7 4 96 6: 13 3 34 | 5 3 99 7: 13 3 38 | 1 0 100 8: 18 5 43 | 0 0 100 9: 201 56 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 28 7 17 | 12 7 17 2048: 56 15 32 | 24 15 33 4096: 112 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff8800b394c800.rpc_stats= snapshot_time: 1746132832.597103752 secs.nsecs start_time: 1746132294.220208649 secs.nsecs elapsed_time: 538.376895103 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 3 6 6 | 0 0 0 2: 1 2 8 | 0 0 0 3: 1 2 10 | 0 0 0 4: 1 2 12 | 0 0 0 5: 1 2 14 | 0 0 0 6: 1 2 16 | 0 0 0 7: 1 2 18 | 0 0 0 8: 1 2 20 | 0 0 0 9: 40 80 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99.sanity-pcc is idle now PASS 99 (66s) == sanity-pcc test 99b: race among unlink | two readers | write | detach for PCC-RO file ========================================================== 16:53:55 (1746132835) 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.281083 s, 746 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: a6cb7f46-2b33-4bf7-a5c3-cebfdcb9cb06 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f99b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a6cb7f46-2b33-4bf7-a5c3-cebfdcb9cb06 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477338 s, 110 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.573493 s, 91.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355193 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243643 s, 2.2 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.494995 s, 106 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.466585 s, 112 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.329239 s, 159 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.741538 s, 70.7 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.414654 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126599 s, 414 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.854806 s, 61.3 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.463956 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495429 s, 1.1 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.43012 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.304716 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038076 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.458291 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283522 s, 185 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309663 s, 1.7 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.427852 s, 123 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.491631 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.61048 s, 85.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229798 s, 2.3 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.554358 s, 94.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.72067 s, 72.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802173 s, 654 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.262746 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.663464 s, 79.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248403 s, 2.1 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672687 s, 77.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.398923 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285731 s, 1.8 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.506185 s, 104 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.422525 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312716 s, 1.7 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.598634 s, 87.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.296792 s, 177 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306273 s, 1.7 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.755198 s, 69.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.360976 s, 145 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239162 s, 2.2 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123269 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.536247 s, 97.8 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238277 s, 2.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.442847 s, 118 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.356596 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270109 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.54066 s, 97.0 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409897 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17237 s, 304 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.62414 s, 84.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.487143 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.460648 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020688 s, 2.5 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.328778 s, 159 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.341247 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.448725 s, 117 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161964 s, 324 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.665563 s, 78.8 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.76286 s, 68.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366925 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.441691 s, 119 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.390812 s, 134 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278522 s, 188 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132397 s, 396 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310472 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.789286 s, 66.4 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.779831 s, 67.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.376955 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.833179 s, 62.9 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.542713 s, 96.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.436545 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202471 s, 2.6 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269083 s, 195 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.522156 s, 100 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266277 s, 2.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515387 s, 102 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.441683 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394755 s, 1.3 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.630741 s, 83.1 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.570315 s, 91.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105148 s, 499 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.630797 s, 83.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134526 s, 390 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.319276 s, 164 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457115 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203818 s, 2.6 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.492006 s, 107 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.920682 s, 56.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.11955 s, 46.8 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.446194 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.627423 s, 83.6 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.631748 s, 83.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265361 s, 2.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.179848 s, 292 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266671 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.635943 s, 82.4 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.567615 s, 92.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.427653 s, 123 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.383855 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.658427 s, 79.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040976 s, 1.3 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211517 s, 2.5 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.482645 s, 109 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.340035 s, 154 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375813 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.411143 s, 128 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00545 s, 52.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.984822 s, 53.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369645 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.11356 s, 47.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00348 s, 52.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393102 s, 1.3 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.676684 s, 77.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.795501 s, 65.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.464221 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323909 s, 1.6 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.580783 s, 90.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.494367 s, 106 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12565 s, 417 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.746859 s, 70.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.816121 s, 64.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.55976 s, 93.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212216 s, 2.5 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.812754 s, 64.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.763181 s, 68.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255535 s, 2.1 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201401 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439871 s, 1.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.662082 s, 79.2 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.810125 s, 64.7 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.607109 s, 86.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221462 s, 2.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.858216 s, 61.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.879875 s, 59.6 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263529 s, 2.0 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.531499 s, 98.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.623717 s, 84.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243861 s, 2.1 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.500681 s, 105 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.476774 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372115 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.669089 s, 78.4 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.583222 s, 89.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.26388 s, 199 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239418 s, 219 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.830584 s, 63.1 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246732 s, 2.1 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.300936 s, 174 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.31089 s, 169 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.738267 s, 71.0 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.334506 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267662 s, 2.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477538 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.334686 s, 157 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378755 s, 1.4 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129022 s, 406 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.63528 s, 82.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.786733 s, 66.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.583611 s, 89.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250203 s, 2.1 GB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800a8a06800.rpc_stats= snapshot_time: 1746132898.023346365 secs.nsecs start_time: 1746132292.570632977 secs.nsecs elapsed_time: 605.452713388 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1015 5 5 | 3 0 0 2: 12 0 5 | 0 0 0 4: 9 0 5 | 0 0 0 8: 14 0 5 | 0 0 0 16: 3507 17 22 | 0 0 0 32: 361 1 24 | 0 0 0 64: 564 2 27 | 0 0 0 128: 1097 5 32 | 0 0 0 256: 13410 67 100 | 5800 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 10132 50 50 | 408 7 7 2: 2045 10 60 | 995 17 24 3: 1074 5 66 | 1393 24 48 4: 966 4 71 | 1096 18 67 5: 852 4 75 | 761 13 80 6: 733 3 79 | 475 8 88 7: 678 3 82 | 287 4 93 8: 633 3 85 | 184 3 96 9: 2746 13 99 | 195 3 99 10: 15 0 99 | 9 0 100 11: 10 0 99 | 0 0 100 12: 10 0 99 | 0 0 100 13: 9 0 99 | 0 0 100 14: 8 0 99 | 0 0 100 15: 7 0 99 | 0 0 100 16: 6 0 99 | 0 0 100 17: 5 0 99 | 0 0 100 18: 5 0 99 | 0 0 100 19: 4 0 99 | 0 0 100 20: 4 0 99 | 0 0 100 21: 4 0 99 | 0 0 100 22: 4 0 99 | 0 0 100 23: 3 0 99 | 0 0 100 24: 3 0 99 | 0 0 100 25: 3 0 99 | 0 0 100 26: 2 0 99 | 0 0 100 27: 2 0 99 | 0 0 100 28: 2 0 99 | 0 0 100 29: 2 0 99 | 0 0 100 30: 2 0 99 | 0 0 100 31: 20 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 285 1 1 | 119 2 2 1: 77 0 1 | 0 0 2 2: 148 0 2 | 0 0 2 4: 10 0 2 | 0 0 2 8: 4 0 2 | 0 0 2 16: 141 0 3 | 0 0 2 32: 249 1 4 | 0 0 2 64: 425 2 6 | 0 0 2 128: 786 3 10 | 0 0 2 256: 415 2 12 | 116 1 4 512: 825 4 16 | 232 3 8 1024: 1571 7 24 | 464 7 16 2048: 3004 15 39 | 928 15 32 4096: 5878 29 69 | 1856 31 64 8192: 6171 30 100 | 2088 35 100 osc.lustre-OST0000-osc-ffff8800b394c800.rpc_stats= snapshot_time: 1746132898.023497314 secs.nsecs start_time: 1746132294.219845666 secs.nsecs elapsed_time: 603.803651648 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 4 8 8 | 0 0 0 2: 2 4 12 | 0 0 0 3: 1 2 14 | 0 0 0 4: 1 2 16 | 0 0 0 5: 1 2 18 | 0 0 0 6: 1 2 20 | 0 0 0 7: 1 2 22 | 0 0 0 8: 1 2 24 | 0 0 0 9: 38 76 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800a8a06800.rpc_stats= snapshot_time: 1746132898.023560126 secs.nsecs start_time: 1746132292.571201345 secs.nsecs elapsed_time: 605.452358781 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 0 0 0 | 0 0 1 32: 0 0 0 | 0 0 1 64: 0 0 0 | 0 0 1 128: 0 0 0 | 0 0 1 256: 352 99 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 36 10 10 | 8 5 5 2: 25 7 17 | 55 35 41 3: 18 5 22 | 50 32 73 4: 18 5 27 | 27 17 91 5: 13 3 30 | 7 4 96 6: 13 3 34 | 5 3 99 7: 13 3 38 | 1 0 100 8: 18 5 43 | 0 0 100 9: 201 56 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 28 7 17 | 12 7 17 2048: 56 15 32 | 24 15 33 4096: 112 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff8800b394c800.rpc_stats= snapshot_time: 1746132898.023652276 secs.nsecs start_time: 1746132294.220208649 secs.nsecs elapsed_time: 603.803443627 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 3 6 6 | 0 0 0 2: 1 2 8 | 0 0 0 3: 1 2 10 | 0 0 0 4: 1 2 12 | 0 0 0 5: 1 2 14 | 0 0 0 6: 1 2 16 | 0 0 0 7: 1 2 18 | 0 0 0 8: 1 2 20 | 0 0 0 9: 40 80 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99b.sanity-pcc is idle now PASS 99b (65s) == sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 16:55:00 (1746132900) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.124592 s, 505 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: d6709f8d-35b6-41ac-8533-777e96e30de1 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d6709f8d-35b6-41ac-8533-777e96e30de1 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.462025 s, 113 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0382581 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0170976 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.116026 s, 280 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.527941 s, 99.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.287777 s, 182 MB/s 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0202459 s, 363 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.422611 s, 124 MB/s 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.146572 s, 42.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0219923 s, 47.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443708 s, 118 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.368459 s, 142 MB/s 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.175308 s, 257 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0104392 s, 100 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.607303 s, 86.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.33437 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.503543 s, 104 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0032716 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.596499 s, 87.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.373153 s, 141 MB/s 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.149189 s, 21.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.522453 s, 100 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369093 s, 142 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0152219 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456945 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250732 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248165 s, 2.1 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.618566 s, 84.8 MB/s 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.169747 s, 37.1 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0139563 s, 75.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445451 s, 118 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0628331 s, 0.0 kB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0321903 s, 32.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.661568 s, 79.2 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0273077 s, 0.0 kB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0192303 s, 54.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.436311 s, 120 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 12+0 records in 12+0 records out 12582912 bytes (13 MB) copied, 0.211656 s, 59.4 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00785628 s, 133 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.128255 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.43573 s, 120 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231409 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171446 s, 3.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+1 records in 0+1 records out 417792 bytes (418 kB) copied, 0.00124909 s, 334 MB/s 20+1 records in 20+1 records out 21561344 bytes (22 MB) copied, 0.0218082 s, 989 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176173 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.568807 s, 92.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217615 s, 2.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 41+0 records in 41+0 records out 42991616 bytes (43 MB) copied, 0.367878 s, 117 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.143201 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.632446 s, 82.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.378299 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181132 s, 289 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.01168 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0307795 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.5486 s, 95.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.356799 s, 147 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.146747 s, 286 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00322951 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672114 s, 78.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.500983 s, 105 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.648313 s, 80.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0262966 s, 39.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401939 s, 130 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342829 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337221 s, 1.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.496581 s, 106 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.354519 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164174 s, 319 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0133535 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.579224 s, 90.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.366739 s, 143 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03631 s, 1.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00360062 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.406905 s, 129 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 48+0 records in 48+0 records out 50331648 bytes (50 MB) copied, 0.371942 s, 135 MB/s 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0239206 s, 219 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.480922 s, 109 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.366337 s, 143 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932863 s, 562 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.498178 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.237268 s, 221 MB/s dd: warning: partial read (188416 bytes); suggest iflag=fullblock Detach /mnt/lustre/f100.sanity-pcc ... 9+2 records in 9+2 records out 10485760 bytes (10 MB) copied, 0.0250654 s, 418 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.449031 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234551 s, 2.2 GB/s 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0786642 s, 93.3 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00709031 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.432423 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.42121 s, 124 MB/s 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.0155991 s, 538 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.472094 s, 111 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 18+0 records in 18+0 records out 18874368 bytes (19 MB) copied, 0.285346 s, 66.1 MB/s dd: warning: partial read (217088 bytes); suggest iflag=fullblock 2+2 records in 2+2 records out 3145728 bytes (3.1 MB) copied, 0.0156839 s, 201 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.37918 s, 138 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.299342 s, 28.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0132059 s, 79.4 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.506895 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11853 s, 442 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 23+1 records in 23+1 records out 24350720 bytes (24 MB) copied, 0.205683 s, 118 MB/s 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.00777543 s, 405 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.349412 s, 150 MB/s 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.313038 s, 6.7 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0202033 s, 51.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.0904455 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.526012 s, 99.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227814 s, 2.3 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.114592 s, 293 MB/s 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.102517 s, 235 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672466 s, 78.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.399051 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.312298 s, 168 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0352691 s, 0.0 kB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0248748 s, 42.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.520539 s, 101 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.473272 s, 111 MB/s 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.125986 s, 358 MB/s 13+0 records in 13+0 records out 13631488 bytes (14 MB) copied, 0.0189772 s, 718 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.584604 s, 89.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.754417 s, 69.5 MB/s 42+0 records in 42+0 records out 44040192 bytes (44 MB) copied, 0.179729 s, 245 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.490826 s, 107 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.385549 s, 136 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0688551 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.0262237 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.439309 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210537 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385986 s, 1.4 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0021518 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.574194 s, 91.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.502931 s, 104 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329884 s, 1.6 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0102512 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.475913 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.379872 s, 138 MB/s 47+0 records in 47+0 records out 49283072 bytes (49 MB) copied, 0.162564 s, 303 MB/s 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.018734 s, 448 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.430066 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224547 s, 233 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 9+0 records in 9+0 records out 9437184 bytes (9.4 MB) copied, 0.0238862 s, 395 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.735658 s, 71.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.575353 s, 91.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.249202 s, 135 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.600579 s, 87.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314942 s, 1.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.10962 s, 9.6 MB/s 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.135835 s, 262 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.555027 s, 94.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.471337 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477097 s, 110 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.518796 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.37745 s, 139 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322134 s, 1.6 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00687702 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.544481 s, 96.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348744 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.29636 s, 177 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.565703 s, 92.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.423477 s, 124 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 42+0 records in 42+0 records out 44040192 bytes (44 MB) copied, 0.208094 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.50139 s, 105 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.370594 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112598 s, 466 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00851246 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.372549 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.648109 s, 80.9 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0100778 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 11+0 records in 11+0 records out 11534336 bytes (12 MB) copied, 0.0365447 s, 316 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.701573 s, 74.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351716 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247694 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035281 s, 1.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.014461 s, 435 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.196143 s, 267 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.502954 s, 104 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.114718 s, 329 MB/s 26+0 records in 26+0 records out 27262976 bytes (27 MB) copied, 0.106312 s, 256 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.525627 s, 99.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.314911 s, 166 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159879 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.37543 s, 140 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.424117 s, 124 MB/s 41+0 records in 41+0 records out 42991616 bytes (43 MB) copied, 0.162179 s, 265 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00876038 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.723569 s, 72.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.430774 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233559 s, 224 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0379921 s, 27.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.154232 s, 184 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.720758 s, 72.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219769 s, 2.4 GB/s 0+1 records in 0+1 records out 786432 bytes (786 kB) copied, 0.0557314 s, 14.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0154691 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.315425 s, 120 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.581303 s, 90.2 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0635135 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00600632 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.491965 s, 107 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.163715 s, 44.8 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0168974 s, 62.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.62546 s, 83.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.48304 s, 109 MB/s 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.121144 s, 199 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.394712 s, 133 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.675171 s, 77.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.268765 s, 195 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0121831 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0028032 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.570329 s, 91.9 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now PASS 100 (66s) == sanity-pcc test 101a: Test auto attach in mount namespace (simulated container) ========================================================== 16:56:06 (1746132966) creating user namespace for 500 sanityu+ 19291 0.0 0.0 108060 368 pts/0 S+ 16:56 0:00 sleep 600 root 19293 0.0 0.0 114880 984 pts/0 S+ 16:56 0:00 grep sleep Created NS: child (sleep) pid=19291 in 0 seconds running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newuidmap] [19291] [0] [500] [1] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newgidmap] [19291] [0] [500] [1] 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707797 s, 741 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d101a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a4d7796c-cd5e-4b83-a161-d666e25fa90a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f101a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a4d7796c-cd5e-4b83-a161-d666e25fa90a (extents) (64bit) (large files) (huge files) Check for temp file in PCC mount Check for temp file in PCC mount from inside namespace Starting copytool 'agt1' on 'oleg353-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d101a.sanity-pcc/d101a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach from inside mount namespace 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00758065 s, 135 kB/s file: /mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc, type: readwrite, PCC_file: /d101a.sanity-pcc/5981/0x2000059f1:0x70:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b394c800.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.000764903 s, 1.3 MB/s Verify auto attach at open from inside NS for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8a06800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b394c800.lru_size=clear Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d101a.sanity-pcc is idle now PASS 101a (8s) == sanity-pcc test 102: PCC-RO should not hange for io_uring I/O engine ========================================================== 16:56:14 (1746132974) SKIP: sanity-pcc test_102 Client OS does not support io_uring I/O engine SKIP 102 (2s) == sanity-pcc test 203: Verify attach/hit bytes statistics data ========================================================== 16:56:16 (1746132976) 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.019266 s, 544 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d203.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/10240 done Creating filesystem with 10240 1k blocks and 2560 inodes Filesystem UUID: 1cad0046-abaf-4cf2-841d-e4da1e3ae792 Superblock backups stored on blocks: 8193 Allocating group tables: 0/2 done Writing inode tables: 0/2 done Creating journal (2048 blocks): done Writing superblocks and filesystem accounting information: 0/2 done /tmp/f203.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1cad0046-abaf-4cf2-841d-e4da1e3ae792 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d203.sanity-pcc/d203.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00795546 s, 129 kB/s ==== stats ==== llite.lustre-ffff8800a8a06800.stats= snapshot_time 1746132977.973861137 secs.nsecs start_time 1746132977.881583471 secs.nsecs elapsed_time 0.092277666 secs.nsecs read_bytes 1 samples [bytes] 1024 1024 1024 1048576 write_bytes 1 samples [bytes] 1024 1024 1024 1048576 read 3 samples [usecs] 234 4275 4786 18407110 write 1 samples [usecs] 3294 3294 3294 10850436 open 3 samples [usecs] 20 22292 22339 496934393 close 3 samples [usecs] 2735 5562 11738 50256550 getattr 2 samples [usecs] 149 4241 4390 18008282 mknod 1 samples [usecs] 8246 8246 8246 67996516 inode_permission 7 samples [usecs] 2 3826 4338 14776624 opencount 3 samples [reqs] 1 3 6 14 openclosetime 2 samples [usecs] 11515 21974 33489 615451901 pcc_attach 1 samples [reqs] pcc_hit_bytes 2 samples [bytes] 1024 1024 2048 2097152 pcc_attach_bytes 1 samples [bytes] 1024 1024 1024 1048576 llite.lustre-ffff8800b394c800.stats= snapshot_time 1746132977.974000239 secs.nsecs start_time 1746132977.881650593 secs.nsecs elapsed_time 0.092349646 secs.nsecs attach_num: 1, attach_bytes: 1024, hit_num: 2, hit_bytes: 2048 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d203.sanity-pcc is idle now PASS 203 (5s) == sanity-pcc test 204a: pin/unpin pcc flag ============== 16:56:21 (1746132981) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819085 s, 640 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1ec7d965-c857-49f9-945c-bf4b86d47743 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f204a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1ec7d965-c857-49f9-945c-bf4b86d47743 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204a.sanity-pcc is idle now PASS 204a (5s) == sanity-pcc test 204b: Permission check for the pin/unpin operation ========================================================== 16:56:26 (1746132986) mdt.lustre-MDT0000.enable_pin_gid=0 /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh: illegal option -- p running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc trusted.pin="[hsm: 100]" running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] unpin: cannot unpin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted mdt.lustre-MDT0000.enable_pin_gid=-1 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=500 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=0 PASS 204b (6s) == sanity-pcc test 204c: PCC pin/unpin without attach ID specified ========================================================== 16:56:32 (1746132992) pin: failed to get id for '/mnt/lustre/f204c.sanity-pcc': No such file or directory 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.093041 s, 676 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=21,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12139) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg353-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg353-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51278/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.203.53,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.203.153@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.203.153@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 5830255e-7464-4886-ad97-b23965f13ba8 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f204c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5830255e-7464-4886-ad97-b23965f13ba8 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204c.sanity-pcc trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204c.sanity-pcc is idle now PASS 204c (5s) == sanity-pcc test complete, duration 1988 sec =========== 16:56:37 (1746132997) === sanity-pcc: start cleanup 16:56:38 (1746132998) === Stopping clients: oleg353-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg353-client.virtnet /mnt/lustre2 opts: === sanity-pcc: finish cleanup 16:56:42 (1746133002) ===