-----============= acceptance-small: sanity-pcc ============----- Thu May 1 18:54:31 EDT 2025 mgs: CentOS Linux release 7.9.2009 (Core) MGS_OS_VERSION_ID=7 MGS_OS_ID=centos MGS_OS_VERSION_CODE=117440512 MGS_OS_ID_LIKE=rhel fedora centos mds1: CentOS Linux release 7.9.2009 (Core) MDS1_OS_ID_LIKE=rhel fedora centos MDS1_OS_ID=centos MDS1_OS_VERSION_ID=7 MDS1_OS_VERSION_CODE=117440512 ost1: CentOS Linux release 7.9.2009 (Core) OST1_OS_VERSION_CODE=117440512 OST1_OS_VERSION_ID=7 OST1_OS_ID_LIKE=rhel fedora centos OST1_OS_ID=centos client: CentOS Linux release 7.9.2009 (Core) CLIENT_OS_ID=centos CLIENT_OS_ID_LIKE=rhel fedora centos CLIENT_OS_VERSION_ID=7 CLIENT_OS_VERSION_CODE=117440512 === sanity-pcc: start setup 18:54:38 (1746140078) === Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) oleg252-client.virtnet: executing check_config_client /mnt/lustre oleg252-client.virtnet: Checking config lustre mounted on /mnt/lustre Checking servers environments Checking clients oleg252-client.virtnet environments Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b6cd5000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012a4c4000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6cd5000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a4c4000.idle_timeout=debug disable quota as required oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all === sanity-pcc: finish setup 18:54:52 (1746140092) === running as uid/gid/euid/egid 500/500/500/500, groups: 500 [true] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d0_runas_test/f7016] nobody:x:99: oleg252-server: ls: cannot access /home/green/git/lustre-release/lustre/tests/except/sanity-pcc.*ex: No such file or directory excepting tests: Killing existing copytools on oleg252-client.virtnet pkill: pidfile not valid Try `pkill --help' for more information. Set HSM on and start Set sanity-hsm HSM policy == sanity-pcc test 1a: Test manual lfs pcc attach with manual HSM restore ========================================================== 18:55:16 (1746140116) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106796 s, 491 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 122e654a-bf66-4215-aadc-a38ff5e6a8e6 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=122e654a-bf66-4215-aadc-a38ff5e6a8e6 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1a.sanity-pcc/d1a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0677873 s, 113 MB/s Restore testing... Waiting 200s for 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1a.sanity-pcc/f1a.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1a.sanity-pcc is idle now PASS 1a (11s) == sanity-pcc test 1b: Test manual lfs pcc attach with restore on remote access ========================================================== 18:55:26 (1746140126) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10056 s, 521 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8d62ae1c-af73-4d57-beac-7f3c9d1a02d6 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8d62ae1c-af73-4d57-beac-7f3c9d1a02d6 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1b.sanity-pcc/d1b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.059915 s, 128 MB/s Restore testing... Waiting 200s for 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1b.sanity-pcc/f1b.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1b.sanity-pcc is idle now PASS 1b (13s) == sanity-pcc test 1c: Test automated attach using Project ID with manual HSM restore ========================================================== 18:55:39 (1746140139) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a97e9800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012aae7000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a97e9800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012aae7000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100349 s, 522 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 116946bc-657a-49d1-8c88-d7d6e3d81679 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=116946bc-657a-49d1-8c88-d7d6e3d81679 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1c.sanity-pcc/d1c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 1.25697 s, 6.1 MB/s sanity-pcc test_1c: @@@@@@ FAIL: expected /mnt/pcc.d1c.sanity-pcc/d1c.sanity-pcc/0bd3/0x200000bd1:0x2:0x0 size: 7654321 got: 0 Trace dump: = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7249:error() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:92:check_file_size() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:102:check_lpcc_sizes() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:357:lpcc_rw_test() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:426:test_1c() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7622:run_one() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7685:run_one_logged() = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7503:run_test() = /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh:428:main() Dumping lctl log to /tmp/testlogs//sanity-pcc.test_1c.*.1746140212.log Cleanup PCC backend on /mnt/lustre rsync: chown "/tmp/testlogs/.sanity-pcc.test_1c.debug_log.oleg252-server.1746140212.log.Nm7WJF" failed: Operation not permitted (1) rsync: chown "/tmp/testlogs/.sanity-pcc.test_1c.dmesg.oleg252-server.1746140212.log.i3I3So" failed: Operation not permitted (1) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1651) [generator=3.1.2] /mnt/pcc.d1c.sanity-pcc is idle now FAIL 1c (77s) == sanity-pcc test 1d: Test Project ID with remote access ========================================================== 18:56:59 (1746140219) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a9890800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012a495800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a9890800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a495800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123234 s, 425 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 384abeea-7dcf-4e3a-8c01-e26a1629febd Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=384abeea-7dcf-4e3a-8c01-e26a1629febd (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1d.sanity-pcc/d1d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0982521 s, 77.9 MB/s Restore testing... Attach and detach testing Start to detach the /mnt/lustre/d1d.sanity-pcc/f1d.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1d.sanity-pcc is idle now PASS 1d (82s) == sanity-pcc test 1e: Test RW-PCC with non-root user ==== 18:58:21 (1746140301) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_1e/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_1e/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00662818 s, 154 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00710209 s, 144 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00653751 s, 157 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] touch: cannot touch '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre PASS 1e (11s) == sanity-pcc test 1f: Test auto RW-PCC cache with non-root user ========================================================== 18:58:32 (1746140312) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a976e000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012e314800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a976e000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012e314800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119099 s, 440 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8969ab80-87ec-44d6-84ea-d15939cc3764 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/71/7 done /tmp/f1f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8969ab80-87ec-44d6-84ea-d15939cc3764 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00666613 s, 154 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00270375 s, 379 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=256] [count=1] 1+0 records in 1+0 records out 256 bytes (256 B) copied, 0.00345191 s, 74.2 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] touch: cannot touch '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1f.sanity-pcc is idle now PASS 1f (75s) == sanity-pcc test 1g: General permission test for RW-PCC ========================================================== 18:59:47 (1746140387) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964777 s, 543 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3037de88-7ec5-45a5-ba0f-27fbbb7bd553 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3037de88-7ec5-45a5-ba0f-27fbbb7bd553 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1g.sanity-pcc/d1g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0129233 s, 79.2 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0115382 s, 88.7 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] detach: cannot detach '/mnt/lustre/f1g.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] Waiting 200s for 'SUCCEED' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00398533 s, 257 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1g.sanity-pcc is idle now PASS 1g (7s) == sanity-pcc test 2a: Test multi open when creating ===== 18:59:54 (1746140394) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a968d000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b63cf800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a968d000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b63cf800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132732 s, 395 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ad934c18-c4db-4cf8-af22-3c48fe1b3342 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ad934c18-c4db-4cf8-af22-3c48fe1b3342 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2a.sanity-pcc/multiop vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.24587 client oleg252-client.virtnet multiop_bg started multiop_pid=24599 ---------- 1 root root 14 May 1 19:01 /mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc/2343/0x200002341:0x2:0x0 Stopping multiop_pid=24599 (kill 24599 on oleg252-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d2a.sanity-pcc is idle now PASS 2a (80s) == sanity-pcc test 2b: Test multi remote open when creating ========================================================== 19:01:14 (1746140474) SKIP: sanity-pcc test_2b Need 2 or more clients, have 1 SKIP 2b (2s) == sanity-pcc test 2c: Test multi open on different mount points when creating ========================================================== 19:01:16 (1746140476) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b632c000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012e14d800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b632c000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012e14d800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128304 s, 409 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ce671e34-66ef-4225-bb56-76821d41c817 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ce671e34-66ef-4225-bb56-76821d41c817 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2c.sanity-pcc/d2c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2c.sanity-pcc/f2c.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.27775 client oleg252-client.virtnet multiop_bg started multiop_pid=27785 Stopping multiop_pid=27785 (kill 27785 on oleg252-client.virtnet) multiopen_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d2c.sanity-pcc is idle now PASS 2c (63s) == sanity-pcc test 3a: Repeat attach/detach operations === 19:02:19 (1746140539) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_3a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0137678 s, 74.4 kB/s Start to RW-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RW-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Start to RO-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RO-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Cleanup PCC backend on /mnt/lustre PASS 3a (5s) == sanity-pcc test 3b: Repeat attach/detach operations on multiple clients ========================================================== 19:02:24 (1746140544) SKIP: sanity-pcc test_3b Need 3 or more clients, have 1 SKIP 3b (2s) == sanity-pcc test 4: Auto cache test for mmap =========== 19:02:26 (1746140546) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a968f800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800a98c3000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a968f800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a98c3000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130551 s, 402 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d4.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 46d56318-cbe2-46ba-bd99-24e08b7ad2a0 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f4.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=46d56318-cbe2-46ba-bd99-24e08b7ad2a0 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d4.sanity-pcc/d4.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' mmap test1: basic mmap operation (PASS, 0.025655s) mmap test2: MAP_PRIVATE not write back (PASS, 0.024647s) mmap test3: concurrent mmap ops on two nodes (PASS, 2.0351s) mmap test4: c1 write to f1 from mmapped f2, c2 write to f1 from mmapped f1 (PASS, 4.0953s) mmap test5: read/write file to/from the buffer which mmapped to just this file (PASS, 0.020464s) mmap test6: check mmap write/read content on two nodes (PASS, 0.36438s) mmap test7: file i/o with an unmapped buffer (SKIPPED, 0s) mmap test8: SIGBUS for beyond file size (SKIPPED, 0s) mmap test9: SIGBUS for negative file offset (SKIPPED, 0s) mmap test10: multi-thread mmap access (PASS, 0.13514s) mmap test11: multi-thread shared mmap (PASS, 0.12259s) mmap test10: mtime not change for readonly mmap access (PASS, 1.0164s) ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a968f800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a98c3000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d4.sanity-pcc is idle now PASS 4 (57s) == sanity-pcc test 5: Mmap & cat a RW-PCC cached file ==== 19:03:23 (1746140603) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888782 s, 590 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d5.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 795084f8-830b-4baa-933c-ff75271d9dd0 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f5.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=795084f8-830b-4baa-933c-ff75271d9dd0 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d5.sanity-pcc/d5.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d5.sanity-pcc is idle now PASS 5 (8s) == sanity-pcc test 6: Test mmap write on RW-PCC ========== 19:03:31 (1746140611) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920834 s, 569 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d6.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 53fd2b76-f470-4128-8b9f-612462367bdc Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f6.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=53fd2b76-f470-4128-8b9f-612462367bdc (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d6.sanity-pcc/d6.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d6.sanity-pcc is idle now PASS 6 (7s) == sanity-pcc test 7a: Fake file detached between fault() and page_mkwrite() for RW-PCC ========================================================== 19:03:38 (1746140618) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10218 s, 513 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 49ccb430-040e-49e1-9163-ddcc06fccf2a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=49ccb430-040e-49e1-9163-ddcc06fccf2a (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7a.sanity-pcc/d7a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1412 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7a.sanity-pcc is idle now PASS 7a (5s) == sanity-pcc test 7b: Test the race with concurrent mkwrite and detach ========================================================== 19:03:43 (1746140623) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890727 s, 589 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 082ce5e6-c1eb-4544-a729-d5037819c2f9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=082ce5e6-c1eb-4544-a729-d5037819c2f9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7b.sanity-pcc/d7b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1413 fail_val=20 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7b.sanity-pcc is idle now PASS 7b (45s) == sanity-pcc test 8: Test fake -ENOSPC tolerance for RW-PCC ========================================================== 19:04:29 (1746140669) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105981 s, 495 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d8.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9a49c5af-3a72-49b4-88ec-612a1c266c78 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f8.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9a49c5af-3a72-49b4-88ec-612a1c266c78 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d8.sanity-pcc/d8.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d8.sanity-pcc is idle now PASS 8 (6s) == sanity-pcc test 9: Test -ENOSPC tolerance on loop PCC device for RW-PCC ========================================================== 19:04:35 (1746140675) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100674 s, 521 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.9a: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 57e2648e-c81a-41f8-a3e1-89f226e7b1aa Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f9.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=57e2648e-c81a-41f8-a3e1-89f226e7b1aa (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.9a/d9.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.9a/d9.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 2.00489 s, 31.4 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.9a is idle now PASS 9 (7s) == sanity-pcc test 10a: Test RW-PCC with user quota on loop PCC device ========================================================== 19:04:42 (1746140682) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0895875 s, 585 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6e74cf49-033d-4cbb-8c97-9cc59678f4dc Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6e74cf49-033d-4cbb-8c97-9cc59678f4dc (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.181869 s, 86.5 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.172129 s, 91.4 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x26:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10a.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10a.sanity-pcc/uquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 1.21402 s, 25.9 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10a.sanity-pcc is idle now PASS 10a (7s) == sanity-pcc test 10b: Test RW-PCC with group quota on loop PCC device ========================================================== 19:04:49 (1746140689) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891976 s, 588 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2ad5058c-ed23-44bb-82b1-34b87d415fdb Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2ad5058c-ed23-44bb-82b1-34b87d415fdb (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.160346 s, 98.1 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.142229 s, 111 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x2b:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10b.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10b.sanity-pcc/gquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 1.0111 s, 31.1 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10b.sanity-pcc is idle now PASS 10b (6s) == sanity-pcc test 10c: Test RO-PCC with user quota on loop PCC device ========================================================== 19:04:55 (1746140695) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0815251 s, 643 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 07216851-f721-48f0-bd55-f670a3d2166a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=07216851-f721-48f0-bd55-f670a3d2166a (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.131584 s, 120 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.119054 s, 132 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10c.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10c.sanity-pcc is idle now PASS 10c (5s) == sanity-pcc test 10d: Test RO-PCC with group quota on loop PCC device ========================================================== 19:05:00 (1746140700) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882669 s, 594 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 8ae4a108-d247-4aca-a920-e849d4cac035 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8ae4a108-d247-4aca-a920-e849d4cac035 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.166717 s, 94.3 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.20668 s, 76.1 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10d.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10d.sanity-pcc is idle now PASS 10d (6s) == sanity-pcc test 10e: Tolerate -EDQUOT failure when auto PCC-RO attach with user quota ========================================================== 19:05:06 (1746140706) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114426 s, 458 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 481b7d0d-66c8-4cd1-b9f7-5f34b03c5b0a Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=481b7d0d-66c8-4cd1-b9f7-5f34b03c5b0a (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10e.sanity-pcc/d10e.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: uid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.035467 s, 59.1 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.00853751 s, 246 MB/s /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0828544 s, 63.3 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0381713 s, 137 MB/s file: /mnt/lustre/f10e.sanity-pcc, type: none /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10e.sanity-pcc is idle now PASS 10e (6s) == sanity-pcc test 10f: Tolerate -EDQUOT failure when auto PCC-RO attach with group quota ========================================================== 19:05:12 (1746140712) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102151 s, 513 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 469cd26b-1620-4948-88c7-36c879666602 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=469cd26b-1620-4948-88c7-36c879666602 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10f.sanity-pcc/d10f.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: gid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0650612 s, 32.2 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0230503 s, 91.0 MB/s /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0706174 s, 74.2 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0537165 s, 97.6 MB/s file: /mnt/lustre/f10f.sanity-pcc, type: none /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10f.sanity-pcc is idle now PASS 10f (6s) == sanity-pcc test 11: Test attach fault injection with simulated PCC file path ========================================================== 19:05:18 (1746140718) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103982 s, 504 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d11.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e3092368-7ae3-44a2-a56f-00c8eeffdce5 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f11.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e3092368-7ae3-44a2-a56f-00c8eeffdce5 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d11.sanity-pcc/d11.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32d9 Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32db lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3a:0x0]: Permission denied (13) lfs pcc pcc: cannot attach with ID: 2: Permission denied (13) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Permission denied lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3b:0x0]: Invalid argument (22) lfs pcc pcc: cannot attach with ID: 2: Invalid argument (22) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Invalid argument Cleanup PCC backend on /mnt/lustre /mnt/pcc.d11.sanity-pcc is idle now PASS 11 (6s) == sanity-pcc test 12: RW-PCC attach races with concurrent HSM remove ========================================================== 19:05:24 (1746140724) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_12/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1414 fail_val=20 /mnt/lustre/f12.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre PASS 12 (25s) == sanity-pcc test 13a: Test auto RW-PCC create caching for UID/GID rule ========================================================== 19:05:49 (1746140749) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: uid={500} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0040871 s, 251 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00467899 s, 219 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00533813 s, 192 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: gid={500} lfs mkdir: dirstripe error on '/mnt/lustre/d13a.sanity-pcc': stripe already set lfs setdirstripe: cannot create dir '/mnt/lustre/d13a.sanity-pcc': File exists running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00863864 s, 119 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00589237 s, 174 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00693944 s, 148 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre PASS 13a (11s) == sanity-pcc test 13b: Test auto RW-PCC create caching for file name with wildcard ========================================================== 19:06:00 (1746140760) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13b/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13b/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: fname={*.h5 suffix.* Mid*dle} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00529914 s, 193 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/suffix.doc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00483226 s, 212 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/MidPADdle] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00827619 s, 124 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/Midpad] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00834676 s, 123 kB/s Cleanup PCC backend on /mnt/lustre PASS 13b (6s) == sanity-pcc test 13c: Check auto RW-PCC create caching for UID/GID/ProjID/fname rule ========================================================== 19:06:06 (1746140766) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a8ed6000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b632e000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a8ed6000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b632e000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11169 s, 469 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d13c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9822ec6a-0626-43ef-92f2-0da29f52a97d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f13c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9822ec6a-0626-43ef-92f2-0da29f52a97d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100 200}&fname={*.h5},uid={500}&gid={500} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00773394 s, 132 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00546504 s, 187 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00925376 s, 111 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00330219 s, 310 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13c.sanity-pcc/ugidcache] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00753342 s, 136 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d13c.sanity-pcc is idle now PASS 13c (69s) == sanity-pcc test 14: Revocation of the layout lock should detach the file automatically ========================================================== 19:07:15 (1746140835) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_14/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear Cleanup PCC backend on /mnt/lustre PASS 14 (6s) == sanity-pcc test 15: Test auto attach at open when file is still valid cached ========================================================== 19:07:22 (1746140842) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13141 s, 399 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d15.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3ffb98b1-fa7e-438d-8400-7f8a0691348c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f15.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3ffb98b1-fa7e-438d-8400-7f8a0691348c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d15.sanity-pcc/d15.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach for non-root user running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00598351 s, 171 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] file: /mnt/lustre/d15.sanity-pcc/f15.sanity-pcc, type: readwrite, PCC_file: /d15.sanity-pcc/3aa0/0x200003ab1:0x11:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] Verify auto attach at open for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear Waiting 200s for 'SUCCEED' Verify auto attach at open for RO-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d15.sanity-pcc is idle now PASS 15 (10s) == sanity-pcc test 16: Test detach with different options ========================================================== 19:07:32 (1746140852) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894207 s, 586 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d16.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 94bb9513-8d65-42a6-93c6-5daff4abbbb4 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f16.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=94bb9513-8d65-42a6-93c6-5daff4abbbb4 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d16.sanity-pcc/d16.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Test detach for RW-PCC Test for reusing valid PCC cache Test for the default detach Waiting 200s for 'SUCCEED' Test detach for RO-PCC detach_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d16.sanity-pcc is idle now PASS 16 (8s) == sanity-pcc test 17: Test auto attach for layout refresh ========================================================== 19:07:40 (1746140860) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989982 s, 530 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d17.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d2983c1b-5570-4f9b-ab1e-743d342f265b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f17.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d2983c1b-5570-4f9b-ab1e-743d342f265b (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d17.sanity-pcc/d17.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d17.sanity-pcc/d17.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 94 autocache: projid={100} Verify auto attach during IO for truncate Verify auto attach during IO for read/write multiop /mnt/lustre/f17.sanity-pcc vO_r TMPPIPE=/tmp/multiop_open_wait_pipe.19254 client oleg252-client.virtnet multiop_bg started multiop_pid=19264 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear Stopping multiop_pid=19264 (kill 19264 on oleg252-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d17.sanity-pcc is idle now PASS 17 (15s) == sanity-pcc test 18: Verify size correctness after re-attach the file ========================================================== 19:07:55 (1746140875) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830677 s, 631 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d18.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: fb120a46-2661-4a0e-a46f-1c57b72343a8 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f18.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=fb120a46-2661-4a0e-a46f-1c57b72343a8 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d18.sanity-pcc/d18.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d18.sanity-pcc/d18.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.114498 s, 36.6 MB/s file: /mnt/lustre/f18.sanity-pcc, type: readwrite, PCC_file: /d18.sanity-pcc/3aac/0x200003ab1:0x1d:0x0, open_count: 0, flags: 0 file: /mnt/lustre/f18.sanity-pcc, type: none /mnt/lustre/f18.sanity-pcc has size 4194304 OK 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00507115 s, 202 kB/s /mnt/lustre2/f18.sanity-pcc has size 1049600 OK file: /mnt/lustre/f18.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d18.sanity-pcc is idle now PASS 18 (8s) == sanity-pcc test 19: Verify the file re-attach works as expected ========================================================== 19:08:03 (1746140883) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106293 s, 493 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d19.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e38fc102-d4de-4a39-bf30-f6c87360033d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f19.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e38fc102-d4de-4a39-bf30-f6c87360033d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d19.sanity-pcc/d19.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d19.sanity-pcc is idle now PASS 19 (6s) == sanity-pcc test 20: Auto attach works after the inode was once evicted from cache ========================================================== 19:08:09 (1746140889) 120+0 records in 120+0 records out 125829120 bytes (126 MB) copied, 0.227778 s, 552 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d20.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/122880 done Creating filesystem with 122880 1k blocks and 30720 inodes Filesystem UUID: b812842c-ea72-4433-bb9c-a29cd45e9d59 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/15 done Writing inode tables: 0/15 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/15 done /tmp/f20.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b812842c-ea72-4433-bb9c-a29cd45e9d59 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d20.sanity-pcc/d20.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d20.sanity-pcc is idle now PASS 20 (8s) == sanity-pcc test 21a: PCC-RO storing as a plain HSM mirror component for plain layout ========================================================== 19:08:17 (1746140897) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140111 s, 374 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1f094ba2-4b87-42ab-a58e-6b2ea34d7f93 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1f094ba2-4b87-42ab-a58e-6b2ea34d7f93 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21a.sanity-pcc/d21a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Plain layout info before PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 267 0x10b 0x240000400 FLR layout info after PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC attach /mnt/lustre/f21a.sanity-pcc again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC detach '/mnt/lustre/f21a.sanity-pcc' again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21a.sanity-pcc is idle now PASS 21a (6s) == sanity-pcc test 21b: PCC-RO stroing as a plain HSM mirror component for FLR layouts ========================================================== 19:08:23 (1746140903) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100116 s, 524 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a6791c8f-c080-44db-bfe3-135f326279c0 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a6791c8f-c080-44db-bfe3-135f326279c0 (extents) (64bit) (large files) (huge files) FLR layout before PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc' again: /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21b.sanity-pcc is idle now PASS 21b (6s) == sanity-pcc test 21c: Verify HSM release works storing PCC-RO as HSM mirror component ========================================================== 19:08:29 (1746140909) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141924 s, 369 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7fc2f814-bcfc-45d1-8aac-4001ecb8b32f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7fc2f814-bcfc-45d1-8aac-4001ecb8b32f (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21c.sanity-pcc/d21c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' /mnt/lustre/f21c.sanity-pcc: (0x00000009) exists archived, archive_id:2 /mnt/lustre/f21c.sanity-pcc lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 0 lmm_stripe_offset: 0 lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' /mnt/lustre/f21c.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21c.sanity-pcc is idle now PASS 21c (8s) == sanity-pcc test 21d: Write should invalidate PCC-RO caching ========================================================== 19:08:37 (1746140917) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175634 s, 299 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f7f74b32-a61a-48d1-9a95-5760dded2618 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f7f74b32-a61a-48d1-9a95-5760dded2618 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21d.sanity-pcc/d21d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21d.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 270 0x10e 0x280000400 PCC-RO attach '/mnt/lustre/f21d.sanity-pcc': /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 4294967296 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Write invalidated PCC-RO cache: /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 4294967296 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21d.sanity-pcc is idle now PASS 21d (6s) == sanity-pcc test 21e: Truncate should invalidate PCC-RO caching ========================================================== 19:08:43 (1746140923) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11028 s, 475 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 956a609e-a96d-4601-8255-8f0164d51604 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=956a609e-a96d-4601-8255-8f0164d51604 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21e.sanity-pcc/d21e.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21e.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 271 0x10f 0x240000400 PCC-RO attach '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Trucate invalidate PCC-RO file '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21e.sanity-pcc is idle now PASS 21e (5s) == sanity-pcc test 21f: mmap write should invalidate PCC-RO caching ========================================================== 19:08:48 (1746140928) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871385 s, 602 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2b387a3c-ba9a-440c-b753-74e079875292 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2b387a3c-ba9a-440c-b753-74e079875292 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21f.sanity-pcc/d21f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21f.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 271 0x10f 0x280000400 PCC-RO attach '/mnt/lustre/f21f.sanity-pcc': /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Mmap write invalidate PCC-RO caching: /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21f.sanity-pcc is idle now PASS 21f (6s) == sanity-pcc test 21g: PCC-RO for file under FLR write pending state ========================================================== 19:08:54 (1746140934) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110635 s, 474 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b434cd33-2986-462c-8f4d-f21272e5afb3 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b434cd33-2986-462c-8f4d-f21272e5afb3 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21g.sanity-pcc/d21g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: wp lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp,pccro lcm_layout_gen: 4 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after Write invalidate '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp lcm_layout_gen: 5 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21g.sanity-pcc is idle now PASS 21g (5s) == sanity-pcc test 21h: Extend mirror once file was PCC-RO cached ========================================================== 19:08:59 (1746140939) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12924 s, 406 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21h.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 584098dd-dc64-4c1f-9bff-05a59d65dcbd Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21h.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=584098dd-dc64-4c1f-9bff-05a59d65dcbd (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21h.sanity-pcc/d21h.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after extend a mirror: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } file: /mnt/lustre/f21h.sanity-pcc, type: none FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc' again: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21h.sanity-pcc is idle now PASS 21h (5s) == sanity-pcc test 21i: HSM release increase layout gen, should invalidate PCC-RO cache ========================================================== 19:09:05 (1746140945) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100985 s, 519 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21i.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 85d2ec43-0636-45e1-a00c-d9324b88abde Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21i.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=85d2ec43-0636-45e1-a00c-d9324b88abde (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21i.sanity-pcc/d21i.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21i.sanity-pcc: (0x00000009) exists archived, archive_id:2 Layout after HSM release /mnt/lustre/f21i.sanity-pcc: /mnt/lustre/f21i.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 16 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC state /mnt/lustre/f21i.sanity-pcc: file: /mnt/lustre/f21i.sanity-pcc, type: none Layout after PCC-RO attach /mnt/lustre/f21i.sanity-pcc again: /mnt/lustre/f21i.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 0 obdidx objid objid group 0 278 0x116 0x240000400 PCC state: file: /mnt/lustre/f21i.sanity-pcc, type: readonly, PCC_file: /d21i.sanity-pcc/3a86/0x200003ab1:0x37:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21i.sanity-pcc is idle now PASS 21i (6s) == sanity-pcc test 21j: PCC-RO for encrypted file ======== 19:09:11 (1746140951) SKIP: sanity-pcc test_21j client encryption not supported SKIP 21j (2s) == sanity-pcc test 22: Test RO-PCC attach for the HSM released file ========================================================== 19:09:13 (1746140953) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12387 s, 423 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d22.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 957ffc23-8b7a-429f-9e63-0f61d536d0d1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f22.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=957ffc23-8b7a-429f-9e63-0f61d536d0d1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d22.sanity-pcc/d22.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 2 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x117:0x0] } - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after HSM release /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 16 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc again: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 279 0x117 0x280000400 Layout after PCC-RO detach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 279 0x117 0x280000400 Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d22.sanity-pcc is idle now PASS 22 (10s) == sanity-pcc test 23: Test write/truncate/mmap-write invalidating RO-PCC caching ========================================================== 19:09:23 (1746140963) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947204 s, 554 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d23.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a1c84f8c-5c32-49c7-ae36-3a9d096d5471 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f23.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a1c84f8c-5c32-49c7-ae36-3a9d096d5471 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d23.sanity-pcc/d23.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Write should invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 4294967296 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 4294967296 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Truncate invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Finish trucate operation Mmap write invalidates RO-PCC caching /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 7 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 4294967296 lhm_archive_ver: 4493216720 lhm_archive_uuid: '' Mmap write /mnt/lustre/f23.sanity-pcc via multiop /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 8 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 8594128896 lhm_archive_ver: 10737419264 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d23.sanity-pcc is idle now PASS 23 (6s) == sanity-pcc test 24a: Test RO-PCC with non-root user === 19:09:29 (1746140969) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103518 s, 506 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c5a8ae3b-1a3e-4d35-9cea-6c76a0e99084 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c5a8ae3b-1a3e-4d35-9cea-6c76a0e99084 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: a0 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00814434 s, 126 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00853822 s, 120 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] touch: cannot touch '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24a.sanity-pcc is idle now PASS 24a (5s) == sanity-pcc test 24b: General permission test for RO-PCC ========================================================== 19:09:34 (1746140974) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851127 s, 616 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c7a61833-2cb2-46e5-9bc4-42007fb332c8 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c7a61833-2cb2-46e5-9bc4-42007fb332c8 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24b.sanity-pcc/d24b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00854597 s, 120 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00411056 s, 249 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] detach: cannot detach '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00692029 s, 148 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24b.sanity-pcc is idle now PASS 24b (6s) == sanity-pcc test 25: Tolerate fake read failure for RO-PCC ========================================================== 19:09:40 (1746140980) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863253 s, 607 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d25.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 64100abd-4741-41d2-8aed-d725235043ec Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f25.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=64100abd-4741-41d2-8aed-d725235043ec (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d25.sanity-pcc/d25.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 fail_loc=0 fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d25.sanity-pcc is idle now PASS 25 (5s) == sanity-pcc test 26: Repeat the attach/detach when the file has multiple openers ========================================================== 19:09:45 (1746140985) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644173 s, 814 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d26.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 841bff52-b517-45c2-8c7a-bae956d1acb1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f26.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=841bff52-b517-45c2-8c7a-bae956d1acb1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d26.sanity-pcc/d26.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2538 client oleg252-client.virtnet multiop_bg started multiop_pid=2549 Stopping multiop_pid=2549 (kill 2549 on oleg252-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2653 client oleg252-client.virtnet multiop_bg started multiop_pid=2663 Stopping multiop_pid=2663 (kill 2663 on oleg252-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2768 client oleg252-client.virtnet multiop_bg started multiop_pid=2778 Stopping multiop_pid=2778 (kill 2778 on oleg252-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2883 client oleg252-client.virtnet multiop_bg started multiop_pid=2893 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=2893 (kill 2893 on oleg252-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.3034 client oleg252-client.virtnet multiop_bg started multiop_pid=3044 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=3044 (kill 3044 on oleg252-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d26.sanity-pcc is idle now PASS 26 (30s) == sanity-pcc test 27: Auto attach at open when the file has multiple openers ========================================================== 19:10:15 (1746141015) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120915 s, 434 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d27.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d82fdbe9-66c4-4ec1-a719-38e284eb8f6d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f27.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d82fdbe9-66c4-4ec1-a719-38e284eb8f6d (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d27.sanity-pcc/d27.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.3953 client oleg252-client.virtnet multiop_bg started multiop_pid=3963 Waiting 200s for 'SUCCEED' Stopping multiop_pid=3963 (kill 3963 on oleg252-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4097 client oleg252-client.virtnet multiop_bg started multiop_pid=4107 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear Waiting 200s for 'SUCCEED' Stopping multiop_pid=4107 (kill 4107 on oleg252-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4241 client oleg252-client.virtnet multiop_bg started multiop_pid=4251 Stopping multiop_pid=4251 (kill 4251 on oleg252-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4360 client oleg252-client.virtnet multiop_bg started multiop_pid=4370 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8ed6000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b632e000.lru_size=clear Stopping multiop_pid=4370 (kill 4370 on oleg252-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d27.sanity-pcc is idle now PASS 27 (26s) == sanity-pcc test 28: RW-PCC attach should fail when the file has cluster-wide openers ========================================================== 19:10:41 (1746141041) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994969 s, 527 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d28.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 0144d619-6b65-43a5-96de-0934da0901da Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f28.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0144d619-6b65-43a5-96de-0934da0901da (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d28.sanity-pcc/d28.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.5228 client oleg252-client.virtnet multiop_bg started multiop_pid=5238 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Stopping multiop_pid=5238 (kill 5238 on oleg252-client.virtnet) multiop /mnt/lustre2/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7016 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Cleanup PCC backend on /mnt/lustre /mnt/pcc.d28.sanity-pcc is idle now PASS 28 (10s) == sanity-pcc test 29a: Auto readonly caching on RO-PCC backend for O_RDONLY open ========================================================== 19:10:51 (1746141051) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff88012a49a000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012e14a800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a49a000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012e14a800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104977 s, 499 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5491186a-c79e-4038-8dab-3f86d3ec8567 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5491186a-c79e-4038-8dab-3f86d3ec8567 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 100 P /mnt/lustre/d29a.sanity-pcc/f29a.sanity-pcc Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29a.sanity-pcc is idle now PASS 29a (50s) == sanity-pcc test 29b: Auto PCC-RO attach in atomic_open ========================================================== 19:11:41 (1746141101) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109098 s, 481 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ba29b5e4-c9b9-487a-a6f3-f769cb29cb18 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ba29b5e4-c9b9-487a-a6f3-f769cb29cb18 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d29b.sanity-pcc/d29b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.dat} 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00959227 s, 427 kB/s 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00727093 s, 563 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00851565 s, 481 kB/s vm.drop_caches = 3 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.0122018 s, 336 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29b.sanity-pcc is idle now PASS 29b (6s) == sanity-pcc test 30: Test lctl pcc del command ========= 19:11:47 (1746141107) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108928 s, 481 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d30.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 60f8c407-7e6b-4ba0-864b-8dd366a06c40 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f30.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=60f8c407-7e6b-4ba0-864b-8dd366a06c40 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 3e autocache: projid={100} check file 0:'4289' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'4287' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'428b' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 1:'0x200004281:0x8:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289' Handle the file: 0x200004281:0x8:0x0 Remove non-cached file: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289/0x200004281:0x8:0x0 flags: 1 check file 1:'0x200004281:0x6:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4287' Handle the file: 0x200004281:0x6:0x0 Detach and remove the PCC cached file: 0x200004281:0x6:0x0 check file 1:'0x200004281:0xa:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/428b' Handle the file: 0x200004281:0xa:0x0 Detach and remove the PCC cached file: 0x200004281:0xa:0x0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d30.sanity-pcc is idle now PASS 30 (5s) == sanity-pcc test 31: Test lctl pcc del command with --keep option ========================================================== 19:11:52 (1746141112) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707331 s, 741 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d31.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 553ce5b2-a29c-4efa-80f0-58f75d507d34 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f31.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=553ce5b2-a29c-4efa-80f0-58f75d507d34 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d31.sanity-pcc/d31.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d31.sanity-pcc/d31.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 30 autocache: projid={100} Cleanup PCC backend on /mnt/lustre /mnt/pcc.d31.sanity-pcc is idle now PASS 31 (6s) == sanity-pcc test 32: Test for RO-PCC when PCC copy is deleted ========================================================== 19:11:58 (1746141118) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975328 s, 538 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d32.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 03dbf4de-65ff-4b6f-a263-2e94f0393df6 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f32.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=03dbf4de-65ff-4b6f-a263-2e94f0393df6 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d32.sanity-pcc/d32.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f32.sanity-pcc vo_rc TMPPIPE=/tmp/multiop_open_wait_pipe.11502 client oleg252-client.virtnet multiop_bg started multiop_pid=11512 Stopping multiop_pid=11512 (kill 11512 on oleg252-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d32.sanity-pcc is idle now PASS 32 (12s) == sanity-pcc test 33: Cache rule with comparator (>, =, <) for file size ========================================================== 19:12:10 (1746141130) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121539 s, 431 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d33.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: ec42d512-9062-43bd-b0cc-088f7d232b57 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f33.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=ec42d512-9062-43bd-b0cc-088f7d232b57 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d33.sanity-pcc/d33.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.doc}&size<{1M} file: /mnt/lustre/myfile.doc, type: none file: /mnt/lustre/myfile.doc, type: none Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size<{5M}&size>{3M} Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size={5M 3M} Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d33.sanity-pcc is idle now PASS 33 (8s) == sanity-pcc test 34: Cache rule with comparator (>, <) for Project ID range ========================================================== 19:12:18 (1746141138) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a3dbc000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800b63cb800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a3dbc000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b63cb800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10491 s, 500 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d34.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 62b2cac1-c10c-47ac-8ff9-366c3537ed8e Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f34.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=62b2cac1-c10c-47ac-8ff9-366c3537ed8e (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d34.sanity-pcc/d34.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid>{100} 99 - /mnt/lustre/f34.sanity-pcc 101 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{100} 102 - /mnt/lustre/f34.sanity-pcc 99 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{120}&projid>{110} 105 - /mnt/lustre/f34.sanity-pcc 121 - /mnt/lustre/f34.sanity-pcc 115 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d34.sanity-pcc is idle now PASS 34 (76s) == sanity-pcc test 35: mmap fault test =================== 19:13:34 (1746141214) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12364 s, 424 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d35.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 453002c2-804f-4378-9a86-3bce19d58842 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f35.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=453002c2-804f-4378-9a86-3bce19d58842 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d35.sanity-pcc/d35.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d35.sanity-pcc is idle now PASS 35 (7s) == sanity-pcc test 36a: Stale RW-PCC copy should be deleted after remove the PCC backend ========================================================== 19:13:41 (1746141221) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942613 s, 556 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: f5efc7ed-f78d-4a16-a318-6c043a55855e Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f5efc7ed-f78d-4a16-a318-6c043a55855e (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a52' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc' check file 1:'0x200004a51:0x3:0x0' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52' Handle the file: 0x200004a51:0x3:0x0 Remove non-cached file: /mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52/0x200004a51:0x3:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36a.sanity-pcc is idle now PASS 36a (6s) == sanity-pcc test 36b: Stale RO-PCC copy should be deleted after remove the PCC backend ========================================================== 19:13:47 (1746141227) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933109 s, 562 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7d03d4e7-4057-4c19-a1da-d2d6428b3d9e Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7d03d4e7-4057-4c19-a1da-d2d6428b3d9e (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a54' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc' check file 1:'0x200004a51:0x5:0x0' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54' Handle the file: 0x200004a51:0x5:0x0 Remove non-cached file: /mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54/0x200004a51:0x5:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36b.sanity-pcc is idle now PASS 36b (6s) == sanity-pcc test 37: Multiple readers on a shared file with PCC-RO mode ========================================================== 19:13:53 (1746141233) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825458 s, 635 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d37.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b7d43bba-b051-4a15-a8f8-691763bf7e5b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b7d43bba-b051-4a15-a8f8-691763bf7e5b (extents) (64bit) (large files) (huge files) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100693 s, 521 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d37.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 9783171c-04ea-406d-a331-02dcfa9bee8b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=9783171c-04ea-406d-a331-02dcfa9bee8b (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d37.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} pcc: - pccpath: /mnt/pcc.d37.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} file: /mnt/lustre/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 file: /mnt/lustre2/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 /mnt/pcc.d37.sanity-pcc.2 is idle now /mnt/pcc.d37.sanity-pcc is idle now PASS 37 (8s) == sanity-pcc test 38: Verify LFS pcc state does not trigger prefetch for auto PCC-RO ========================================================== 19:14:01 (1746141241) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b63cd000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012ab83800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b63cd000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012ab83800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818868 s, 640 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d38.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7a1bd775-de6b-46dd-b9e7-b94ec6736566 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f38.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7a1bd775-de6b-46dd-b9e7-b94ec6736566 (extents) (64bit) (large files) (huge files) file: /mnt/lustre/d38.sanity-pcc/f38.sanity-pcc, type: none QQQQQ Cleanup PCC backend on /mnt/lustre /mnt/pcc.d38.sanity-pcc is idle now PASS 38 (52s) == sanity-pcc test 39: Test Project quota on loop PCC device ========================================================== 19:14:53 (1746141293) SKIP: sanity-pcc test_39 Not support project quota on local filesystem SKIP 39 (1s) == sanity-pcc test 40: Test async open attach in the background for PCC-RO file ========================================================== 19:14:55 (1746141295) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg252-client.virtnet /mnt/lustre (opts:) Stopping client oleg252-client.virtnet /mnt/lustre opts: Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg252-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg252-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg252-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg252-server' oleg252-server: oleg252-server.virtnet: executing load_modules_local oleg252-server: Loading modules from /home/green/git/lustre-release/lustre oleg252-server: detected 4 online CPUs by sysfs oleg252-server: MODOPTS_LIBCFS= oleg252-server: Force libcfs to create 2 CPU partitions oleg252-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg252-server: oleg252-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg252-client: oleg252-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Starting client oleg252-client.virtnet: -o user_xattr,flock 192.168.202.152@tcp:/lustre /mnt/lustre2 Started clients oleg252-client.virtnet: 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff88012b636000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012e148000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012b636000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012e148000.idle_timeout=debug disable quota as required 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.311389 s, 673 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d40.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: b1812a15-fb2f-4258-9745-9b5b0544b5c7 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f40.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b1812a15-fb2f-4258-9745-9b5b0544b5c7 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d40.sanity-pcc/d40.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.532328 s, 98.5 MB/s 100 - /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none ldlm.namespaces.lustre-OST0000-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012e148000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012e148000.lru_size=clear osc.lustre-OST0000-osc-ffff88012b636000.stats=clear osc.lustre-OST0000-osc-ffff88012e148000.stats=clear osc.lustre-OST0001-osc-ffff88012b636000.stats=clear osc.lustre-OST0001-osc-ffff88012e148000.stats=clear fail_val=1 fail_loc=0x214 llite.lustre-ffff88012b636000.pcc_async_threshold=5MB llite.lustre-ffff88012e148000.pcc_async_threshold=5MB Test open attach with pcc_async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 2.00618 s, 523 kB/s Before: 0 After: 3 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none, flags: attaching /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 8s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff88012b636000.pcc_async_threshold=1G llite.lustre-ffff88012e148000.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012e148000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012e148000.lru_size=clear osc.lustre-OST0000-osc-ffff88012b636000.stats=clear osc.lustre-OST0000-osc-ffff88012e148000.stats=clear osc.lustre-OST0001-osc-ffff88012b636000.stats=clear osc.lustre-OST0001-osc-ffff88012e148000.stats=clear Test open attach with async_threshold=1G file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: readonly, PCC_file: /d40.sanity-pcc/59f3/0x2000059f1:0x2:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00568252 s, 185 MB/s Before: 50 After: 50 Time1: 2 Time2: 11 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff88012b636000.pcc_async_threshold=5MB llite.lustre-ffff88012e148000.pcc_async_threshold=5MB ldlm.namespaces.lustre-OST0000-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012e148000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012e148000.lru_size=clear Read 1MB data with async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 1.12229 s, 934 kB/s /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 9s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff88012b636000.pcc_async_threshold=1G llite.lustre-ffff88012e148000.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff88012e148000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff88012e148000.lru_size=clear Read 1MB data with async_threshold=1G 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00567827 s, 185 MB/s Time1: 1 Time2: 9 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d40.sanity-pcc is idle now PASS 40 (90s) == sanity-pcc test 41: Test mtime rule for PCC-RO open attach with O_RDONLY mode ========================================================== 19:16:25 (1746141385) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848147 s, 618 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d41.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2f2e1abc-6240-4d19-b087-b2105966aaf2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f41.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2f2e1abc-6240-4d19-b087-b2105966aaf2 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d41.sanity-pcc/d41.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: mtime>{1m} pcc_ro_data pcc_ro_data Cleanup PCC backend on /mnt/lustre /mnt/pcc.d41.sanity-pcc is idle now PASS 41 (5s) == sanity-pcc test 42: PCC attach without attach ID specified ========================================================== 19:16:30 (1746141390) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0808223 s, 778 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d42.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 28dd8c72-b1e3-435c-b3de-3d20d15208ea Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f42.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=28dd8c72-b1e3-435c-b3de-3d20d15208ea (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d42.sanity-pcc/d42.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} file: /mnt/lustre/f42.sanity-pcc, type: readonly, PCC_file: /d42.sanity-pcc/59f5/0x2000059f1:0x4:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d42.sanity-pcc is idle now PASS 42 (4s) == sanity-pcc test 43: Auto attach at open() should add capacity owner check ========================================================== 19:16:35 (1746141395) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0991351 s, 635 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d43.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 0ac83a23-f24c-406a-81fc-53174ea157f4 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f43.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0ac83a23-f24c-406a-81fc-53174ea157f4 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d43.sanity-pcc/d43.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: size<{100M} file: /mnt/lustre/f43.sanity-pcc, type: none running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f43.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f43.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/f43.sanity-pcc] file: /mnt/lustre/f43.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d43.sanity-pcc is idle now PASS 43 (3s) == sanity-pcc test 44: Verify valid auto attach without re-fetching the whole files ========================================================== 19:16:38 (1746141398) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.139488 s, 451 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d44.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 9274d687-631c-41c6-a507-364f2c8ca117 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f44.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9274d687-631c-41c6-a507-364f2c8ca117 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d44.sanity-pcc/d44.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff88012b636000.pcc_async_threshold=0 llite.lustre-ffff88012e148000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.575558 s, 91.1 MB/s /mnt/lustre/f44.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 420 0x1a4 0x240000400 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422009 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.44895 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456041 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.424215 s, 124 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.426823 s, 123 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.474842 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.47541 s, 110 MB/s 52428800 bytes (52 MB) copied, 0.430132 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.451352 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.508836 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.502296 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.47962 s, 109 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.505293 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.474998 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13829 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515634 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.528637 s, 99.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263633 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263078 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487075 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502086 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333995 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347812 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274153 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323294 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670797 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642654 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864329 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405783 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092444 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513241 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405146 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830847 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277763 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029288 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362755 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473571 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026832 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378507 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300976 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489566 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422238 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205022 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024552 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534467 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439432 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247935 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359769 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243917 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277222 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233604 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241656 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030647 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225428 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238416 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382379 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325139 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247639 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219017 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050376 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341391 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500139 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207556 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357817 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337503 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297296 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374195 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348769 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381096 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332953 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055195 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415316 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273616 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323511 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352243 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293707 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219027 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288845 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399552 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376354 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047394 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445911 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478486 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314549 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246952 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373213 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269555 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266383 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207128 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024859 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499074 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375272 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480037 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637965 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248247 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263856 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386787 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253854 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397867 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325003 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492421 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038988 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236909 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276024 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547791 s, 957 MB/s 52428800 bytes (52 MB) copied, 0.0612278 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357989 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384474 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0463445 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074813 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664759 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476293 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280771 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208491 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335604 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192072 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452273 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472806 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477657 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225077 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422974 s, 1.2 GB/s , 0.0478177 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433513 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372957 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0400881 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343219 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242849 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354978 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284599 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272216 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590331 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643431 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651726 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424124 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247075 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439954 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078714 s, 666 MB/s 52428800 bytes (52 MB) copied, 0.0688416 s, 762 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100023 s, 524 MB/s 52428800 bytes (52 MB) copied, 0.0977583 s, 536 MB/s 52428800 bytes (52 MB) copied, 0.0720527 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119806 s, 438 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894431 s, 586 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.080184 s, 654 MB/s , 0.105947 s, 495 MB/s 52428800 bytes (52 MB) copied, 0.104585 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293775 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108578 s, 483 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392021 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649494 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212326 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446909 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310499 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418397 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338113 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049383 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376997 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256953 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331414 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401768 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049953 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271394 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245901 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482022 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443695 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388973 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402088 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414744 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493128 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287386 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036023 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357611 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532272 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226696 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187067 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305806 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196249 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02916 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212056 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368028 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311985 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466141 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037866 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260301 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441261 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367132 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459041 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0328921 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290658 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806758 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369831 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647822 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060206 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143572 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278101 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403236 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034413 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305639 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299806 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335103 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028712 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386748 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564025 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483454 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406794 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301841 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412278 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257288 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202893 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255721 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290904 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273309 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313223 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469019 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037956 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040262 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186223 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231657 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357053 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437525 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445358 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104249 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281176 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432058 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516008 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263955 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629707 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433589 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323266 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456678 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373436 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476265 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225031 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233138 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311715 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358065 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520761 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305383 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463895 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473578 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683911 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666364 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777697 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727023 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041283 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260132 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252558 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226049 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562025 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666148 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302984 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502545 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480707 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546795 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300062 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415568 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223637 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270738 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260264 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313848 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229036 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366714 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402691 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594797 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601274 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460468 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781657 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04785 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028609 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670706 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213143 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250164 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338703 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034302 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521571 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328754 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059974 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199924 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467227 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044107 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635255 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253821 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339784 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420834 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055639 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373241 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638787 s, 821 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0634348 s, 826 MB/s , 0.07772 s, 675 MB/s 52428800 bytes (52 MB) copied, 0.0942209 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618447 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044818 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561564 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613764 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377652 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370045 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035474 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358428 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414415 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484056 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467668 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053233 s, 985 MB/s 52428800 bytes (52 MB) copied, 0.0422705 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532089 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471928 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528799 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622331 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262819 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367966 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382429 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435344 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508278 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244324 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734723 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538909 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503229 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736506 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435578 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480427 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448042 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487627 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272316 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401992 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665542 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414209 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515434 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548535 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552045 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537755 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961214 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487222 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917108 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398978 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445634 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039023 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257157 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297677 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306168 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604562 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443068 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194938 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042109 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375733 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296949 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391853 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512084 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349981 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394945 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046623 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473333 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200308 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522453 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399933 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035482 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029713 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436011 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488259 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021915 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362407 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312234 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328739 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040796 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251289 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575626 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523086 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347048 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522347 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398325 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0266678 s, 2.0 GB/s , 0.0373663 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485605 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621522 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757529 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560448 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379878 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178251 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406474 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118316 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362199 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390441 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208554 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636321 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635835 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507171 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271114 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474019 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218115 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362916 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444652 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361004 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368786 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246181 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044815 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043438 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778706 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040154 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356851 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254254 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197646 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496025 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277022 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222533 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389104 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275885 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251741 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279844 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442612 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163923 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229579 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385023 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365853 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593495 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192419 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167695 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214533 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387352 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249956 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187943 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241198 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287685 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363486 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365785 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422708 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416359 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625737 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276558 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320423 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218585 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245381 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488114 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028815 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179497 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288088 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255285 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377348 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309153 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290836 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288226 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433168 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030565 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436291 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461775 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354052 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290884 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271919 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0231327 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316393 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394513 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284442 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381971 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281362 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064008 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406157 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298532 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459796 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494374 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450712 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345091 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502078 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464979 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029972 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247464 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359807 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396509 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458144 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031725 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455712 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382304 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370753 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517676 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518472 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483113 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305149 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315907 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201375 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242654 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266667 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375709 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351777 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203407 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488783 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340681 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383249 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345408 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338005 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041261 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363397 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206584 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618113 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206285 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332362 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362709 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044449 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031863 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455921 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447981 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024481 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202482 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658471 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612383 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476526 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453765 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618636 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615288 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331015 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260339 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325071 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330213 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027359 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297259 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219967 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395725 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464164 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351103 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515333 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621024 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619258 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756867 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049697 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198006 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246457 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290921 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288401 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245665 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034499 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364858 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033895 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207067 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059294 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671259 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168177 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312613 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169164 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333479 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199175 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294445 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281607 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038556 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020759 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366233 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429528 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199129 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283848 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289239 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222929 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324571 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496802 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434295 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390417 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303741 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578714 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254321 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315136 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226005 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267931 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334849 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376455 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237097 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630774 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253818 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252476 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453269 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281129 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257629 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510222 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250182 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522412 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255272 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400005 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350309 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513595 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322067 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270553 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271776 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273581 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283884 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396096 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356198 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286301 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376259 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249767 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251601 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030926 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018275 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023656 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368271 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316402 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303933 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321969 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406748 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267171 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044595 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644664 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212356 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234412 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330615 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275728 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249811 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522481 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509595 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257895 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290298 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041744 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334837 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017156 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652883 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453648 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232648 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188303 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211106 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305682 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264601 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217145 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463013 s, 1.1 GB/s , 0.0369754 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316521 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403721 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426065 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031507 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0235198 s, 2.2 GB/s , 0.0191763 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219014 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226195 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418291 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057213 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353564 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576827 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325389 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293968 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410095 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351897 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224179 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289259 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235463 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273714 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227627 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171161 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433671 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377061 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198386 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055052 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02715 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332847 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432545 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029021 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380267 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370049 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027604 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283569 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720355 s, 728 MB/s 52428800 bytes (52 MB) copied, 0.0719215 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062533 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263753 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472247 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306168 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226415 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235294 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0300073 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325941 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323597 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262231 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204848 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372916 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262428 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244263 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291689 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387842 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279398 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357753 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045905 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250267 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343803 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0294927 s, 1.8 GB/s , 0.0307088 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460963 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026479 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422641 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257138 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261442 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299362 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489046 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243801 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328395 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371076 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460429 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023571 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448484 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301587 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242946 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240271 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316744 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335352 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275974 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300764 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644899 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332548 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331472 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351932 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275898 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474031 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429102 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067831 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066396 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699911 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601023 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498528 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448452 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982253 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104879 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120639 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565862 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389119 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321193 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264069 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448841 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486171 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322397 s, 1.6 GB/s , 0.0451414 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674873 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382591 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537954 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799317 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685351 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426561 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286239 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045673 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326845 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0298089 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300023 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318709 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365504 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393838 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246492 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272671 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322058 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258207 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341111 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402121 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029393 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336711 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343089 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049194 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257851 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534267 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366098 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554668 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325143 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398692 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042238 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551553 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064292 s, 815 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057463 s, 912 MB/s 52428800 bytes (52 MB) copied, 0.0489534 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369533 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332954 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227559 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314878 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022823 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354156 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041646 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429419 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040199 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670585 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344345 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313159 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449778 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234688 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031266 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364743 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254405 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412596 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025086 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025277 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306651 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384406 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335222 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335878 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390293 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032873 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252966 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430844 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0519615 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498259 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216099 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512846 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321828 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417618 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022581 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198349 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212257 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390598 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029076 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326116 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320506 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255202 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269457 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03228 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426851 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403739 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219611 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288148 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227454 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030857 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289962 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532119 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319408 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247758 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338418 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557885 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410436 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444248 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564629 s, 929 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367288 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403992 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0646451 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380091 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394988 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307008 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705791 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324638 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530761 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489451 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475186 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519749 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295354 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045703 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206177 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656228 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047187 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444111 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258712 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290909 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369013 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419266 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450244 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361658 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397896 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327059 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388407 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378695 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626228 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554677 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498868 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689558 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538146 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434414 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376626 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554115 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376064 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436121 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290557 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347606 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439783 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337138 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399905 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477542 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354929 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367709 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377575 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376131 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710731 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584576 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550058 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917509 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364593 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553108 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496412 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865767 s, 606 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04449 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964005 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376345 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277801 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412837 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486452 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346441 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337711 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400541 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438223 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356399 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305045 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055448 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651208 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0841517 s, 623 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0407895 s, 1.3 GB/s , 0.020298 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601546 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304049 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034152 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315275 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530869 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528376 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609665 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382081 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390972 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272597 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492504 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306352 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289954 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315828 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311102 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252725 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362993 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231549 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037572 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475212 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395069 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309583 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345548 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234582 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383868 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491512 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270373 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450357 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485323 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727211 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412607 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659365 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587934 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212062 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268576 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218472 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299695 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243693 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317542 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320005 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280644 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532521 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051249 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694658 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359028 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386037 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359815 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272139 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413572 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313393 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274598 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353866 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327503 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275403 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341361 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416326 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442307 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320814 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469561 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214129 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462763 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308327 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221276 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341571 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379223 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034364 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508672 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416795 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451883 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074784 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276775 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595338 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492873 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360045 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511666 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0935687 s, 560 MB/s , 0.0389912 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549119 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952981 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0972459 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065222 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229201 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527218 s, 994 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286801 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280943 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444429 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380487 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266885 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461255 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487466 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641805 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030478 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300919 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242626 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335306 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243858 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250438 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272996 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291608 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033416 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619778 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416519 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371346 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594324 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439891 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360065 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266768 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300417 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318509 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03985 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482682 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591175 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352573 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464153 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318234 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347181 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.049265 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317951 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297634 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025003 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299322 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265011 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383552 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346897 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442088 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318423 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287317 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301894 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274669 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266162 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215487 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231654 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257431 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377346 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221452 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296109 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401698 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465749 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292801 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401268 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325763 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231225 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404211 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450373 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222029 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261032 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337195 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320801 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589576 s, 889 MB/s 52428800 bytes (52 MB) copied, 0.0477462 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285297 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274337 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036422 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474176 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027785 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230814 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286655 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286376 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234864 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509681 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497355 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325348 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414309 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424188 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459938 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308998 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04755 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370473 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274018 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438754 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578246 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285359 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331851 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425179 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581805 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702565 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488375 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041782 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029659 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369314 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426163 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392245 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528499 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047753 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276406 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273615 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044557 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04188 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444586 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442524 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434446 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0542235 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481012 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542088 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430148 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339577 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348818 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238368 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318391 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493371 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037028 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251195 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352452 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368735 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504041 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045081 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325871 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371085 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037322 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455727 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02951 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303491 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036491 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439071 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319866 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342141 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218197 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030126 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282737 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257713 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253783 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260299 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350312 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270236 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600846 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275366 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349796 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505089 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255026 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311094 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377717 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294348 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542045 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370559 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100171 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103058 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832958 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821464 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123155 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148195 s, 354 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117848 s, 445 MB/s 52428800 bytes (52 MB) copied, 0.156816 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473741 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468469 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609893 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113688 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205583 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929663 s, 564 MB/s 52428800 bytes (52 MB) copied, 0.0843089 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796025 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435441 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474218 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0573937 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032389 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332353 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366127 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625556 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557904 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358397 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291426 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338795 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369414 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302173 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317249 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342868 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295227 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387393 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311831 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032814 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318683 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218982 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377637 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426298 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308967 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395771 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037363 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02213 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347857 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351358 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032531 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250774 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418918 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490641 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501934 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0232563 s, 2.3 GB/s , 0.0265421 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365077 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342698 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235264 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307202 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379983 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032118 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374097 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439876 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471378 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044393 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500487 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460812 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521164 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514138 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398963 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401575 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262419 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393563 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355725 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371919 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318176 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390326 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401203 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301155 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208535 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261755 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533767 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601621 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514185 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307574 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759728 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589117 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082799 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526932 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311672 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241169 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426891 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327322 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492241 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235954 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255439 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355786 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333634 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309272 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363188 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458951 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407928 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557841 s, 940 MB/s , 0.0372754 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383887 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032374 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052384 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304655 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532819 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342143 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319134 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415448 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262088 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037655 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345089 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406766 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431672 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323032 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316105 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280418 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228768 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266732 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240054 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215326 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253847 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228893 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395221 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356292 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337934 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397925 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299612 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327962 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739799 s, 709 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632492 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499438 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031833 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477737 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254155 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281068 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249177 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278196 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244903 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313963 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270956 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454039 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0359666 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292365 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365222 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218694 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0423486 s, 1.2 GB/s , 0.0278214 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510983 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337797 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298298 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320044 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204436 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226512 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287541 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280226 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217187 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245159 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298608 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272555 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308367 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032682 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285158 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037947 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480169 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359259 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439135 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032319 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393464 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560877 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529706 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580336 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103689 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508414 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838997 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642032 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500802 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254387 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456001 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389022 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584796 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444352 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314226 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263909 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418629 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037549 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292061 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268482 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043336 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515783 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354574 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709882 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503716 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629706 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473117 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376884 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603375 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304031 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281492 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276001 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314253 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258386 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334336 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323333 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406806 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237571 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387353 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256338 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0313726 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434918 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417518 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403206 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0225672 s, 2.3 GB/s , 0.0273369 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0241785 s, 2.2 GB/s , 0.0247274 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215126 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244616 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241292 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233795 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604374 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458929 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374976 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595718 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267695 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068296 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361534 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311416 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278327 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242969 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229694 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290688 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331837 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434157 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493218 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229409 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226848 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244084 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227238 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307536 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387655 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325749 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295944 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369907 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474355 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343202 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261723 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225317 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305836 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265771 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396776 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249326 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244943 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256688 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474514 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371872 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276142 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327805 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422751 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427176 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458877 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0647155 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0614028 s, 854 MB/s 52428800 bytes (52 MB) copied, 0.0515285 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560917 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390397 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568166 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301048 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220996 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289365 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222811 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298981 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295892 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314058 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364395 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328503 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330283 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360419 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438538 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673685 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803871 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712755 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623464 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868885 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449211 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0852185 s, 615 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233154 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345313 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274105 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345653 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267393 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300935 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273544 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444139 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310876 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571931 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595657 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339227 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494527 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431029 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904593 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333029 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578735 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262922 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228511 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352839 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304375 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046495 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447544 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489717 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526362 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462949 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318181 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223129 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307543 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296731 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366512 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331158 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363803 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255079 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344271 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345942 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320883 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356477 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323002 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207608 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250361 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280008 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345207 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216037 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246847 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373697 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327918 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463732 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726417 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479086 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389615 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519228 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675211 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526175 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453739 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453938 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278095 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370734 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283738 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280091 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293113 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426106 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213391 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457287 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405724 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285309 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349955 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260667 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430753 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373307 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388498 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277526 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369941 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356729 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.047301 s, 1.1 GB/s , 0.0522938 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285682 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536264 s, 978 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061479 s, 853 MB/s 52428800 bytes (52 MB) copied, 0.0862182 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300177 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325885 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404642 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026871 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269922 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300274 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309663 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354792 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393428 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352748 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266634 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257186 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429372 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323078 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297651 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268218 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303288 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309039 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419724 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269728 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038952 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542856 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430591 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466572 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030517 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446519 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310631 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250076 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267687 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358112 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240675 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490771 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640954 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523213 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568168 s, 923 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269486 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.034816 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413583 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035276 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344806 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308618 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340443 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371873 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412068 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389928 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022075 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303763 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334117 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484228 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241553 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296765 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029608 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03523 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391592 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562682 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545978 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400183 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360174 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368961 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388093 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519191 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327841 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302735 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022949 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440173 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435729 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395898 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407586 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437872 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487001 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423406 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269011 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289806 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386135 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382394 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196056 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297497 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308493 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252911 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310366 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300624 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303664 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237148 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421917 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216203 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294186 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404437 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809025 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0856392 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365725 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486701 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.046141 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695322 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563787 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611945 s, 857 MB/s , 0.0446351 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441792 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397571 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221732 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352575 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268222 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258305 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399425 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043352 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04857 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470163 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302825 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240912 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398141 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034057 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394114 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038891 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339468 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0656468 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517929 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420974 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456604 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431073 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196082 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347553 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034675 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042007 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265783 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251622 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413206 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461381 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568953 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583047 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662325 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451731 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295989 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347278 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432833 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554302 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448409 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025504 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371195 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254656 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445312 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508441 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324406 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253959 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411177 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341294 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271716 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372017 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394033 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218871 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301354 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366268 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035926 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372659 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486703 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029046 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457969 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494364 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250866 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250493 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400482 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385756 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319099 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225051 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221216 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428369 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625256 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265815 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407074 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305757 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307101 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269805 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503849 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401663 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437737 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353859 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302172 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367575 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464108 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414945 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249098 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391935 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042769 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655162 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571822 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380867 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317387 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310157 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026554 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392811 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033477 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402037 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289726 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351242 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245973 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217223 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236179 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266641 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026506 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208578 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325915 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380013 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233987 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309886 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363725 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266836 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372332 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280018 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244373 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376191 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326069 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452246 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.038931 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294848 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032383 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036124 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443992 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268861 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292065 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405895 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306888 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247911 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366046 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273369 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448037 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477499 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324361 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534825 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218745 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297207 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290427 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234803 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202975 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0234747 s, 2.2 GB/s , 0.0234991 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319849 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267713 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237543 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293881 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379668 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347118 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426629 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359707 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444743 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233471 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274048 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276433 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463339 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389077 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528626 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518409 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573396 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424376 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410237 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552699 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399515 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191482 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210118 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296346 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232076 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286166 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198769 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201161 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310677 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316316 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277646 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262278 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354729 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254864 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245312 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283114 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243744 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313495 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395918 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366745 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497122 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303943 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260821 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280735 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273241 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443035 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562554 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383408 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0640852 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282489 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634035 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419874 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426586 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221821 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334058 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453052 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296147 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031799 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281621 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269825 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255152 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253974 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253825 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426552 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229805 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302405 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293073 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400808 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337011 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306473 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364341 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525671 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510077 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039601 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297228 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258839 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219975 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302024 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02545 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353968 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024429 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299681 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282447 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229693 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515635 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280429 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201287 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311051 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251888 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342017 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250803 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254609 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020094 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325411 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031464 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368089 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378532 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233182 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256541 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273757 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.025633 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211585 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03612 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453884 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502987 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302027 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389258 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284781 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341374 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262054 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301056 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025278 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295452 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258448 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352022 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452538 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308065 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385005 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262128 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317388 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295529 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0266266 s, 2.0 GB/s , 0.0258413 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246226 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347983 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312144 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284942 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535115 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259165 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218438 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242907 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246022 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212251 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336289 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457111 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517283 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403904 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302036 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02729 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463953 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547119 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634158 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492436 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404943 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567888 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071669 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11203 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705033 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423208 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399234 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0914403 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366393 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286498 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0242695 s, 2.2 GB/s , 0.0376282 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290137 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405263 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028867 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214106 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411321 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289028 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399221 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0295645 s, 1.8 GB/s , 0.0301523 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248392 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284567 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029886 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223304 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368672 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401398 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022075 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271024 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434467 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381014 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0242689 s, 2.2 GB/s , 0.0449963 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220949 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444262 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241772 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265882 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307269 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198573 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030435 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289526 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022639 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339081 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405075 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648152 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397212 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624969 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530703 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727625 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389852 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0215266 s, 2.4 GB/s , 0.0270031 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370562 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315951 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373255 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023025 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026092 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355824 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509933 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512809 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502652 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471803 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593161 s, 884 MB/s 52428800 bytes (52 MB) copied, 0.0485012 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373383 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501628 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373794 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384169 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403134 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520673 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236856 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333514 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277963 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021771 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509788 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403174 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232936 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238957 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219969 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257051 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228671 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281834 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216291 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309284 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258537 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026978 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262301 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034794 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195188 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307482 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265953 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299012 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194388 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222202 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276304 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325551 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270448 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212256 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225148 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262967 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208784 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213042 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262965 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307872 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262103 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296801 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251697 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238184 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185304 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274659 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302063 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230273 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026689 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293028 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235706 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243543 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311209 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019238 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513582 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331988 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334123 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409905 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023609 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264435 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282361 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025516 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257121 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227813 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202181 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304361 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397547 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443088 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281247 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235946 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260387 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200737 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271908 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200723 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235443 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238157 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200574 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197523 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286259 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309152 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412328 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399207 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214714 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032227 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214685 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263319 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234873 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249972 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213156 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213363 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323077 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196803 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230798 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261311 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263757 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213694 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023686 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298989 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0362303 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.022464 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0235342 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0272486 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249496 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363572 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247902 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029052 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298754 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270342 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297683 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192648 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181032 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226413 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243385 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259332 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304529 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270802 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274797 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026855 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337271 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214236 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191782 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209193 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215277 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260734 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248036 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208777 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207165 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024778 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243476 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168089 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231741 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181683 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275712 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297642 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325094 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348857 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269376 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354778 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283395 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191067 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208008 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169867 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016751 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295504 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217165 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228652 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195069 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209437 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186871 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270059 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220017 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226483 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316518 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217583 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223483 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226888 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283918 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231375 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223918 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282244 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240439 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221636 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177671 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154995 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205841 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161382 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186514 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222636 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236529 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021726 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175343 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210404 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183257 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216697 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246697 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264678 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298933 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193719 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255285 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236866 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247063 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186367 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194401 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230453 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198387 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201597 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0200487 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0202298 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208326 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241992 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022704 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287703 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311779 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202146 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015958 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0173964 s, 3.0 GB/s , 0.0185335 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157479 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172683 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0197758 s, 2.7 GB/s , 0.0206243 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0221542 s, 2.4 GB/s , 0.0221512 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162291 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238146 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190419 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242326 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208102 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159794 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177346 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194469 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219509 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222603 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204008 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223353 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218863 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169988 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210856 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177552 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178594 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.018334 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207231 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261291 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345685 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018696 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290428 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176184 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0158764 s, 3.3 GB/s , 0.0164858 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162633 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198349 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181903 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157771 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169028 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189197 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236586 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201804 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018557 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0179306 s, 2.9 GB/s , 0.0274013 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162885 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018811 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200848 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218934 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299197 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029151 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361433 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194511 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301584 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266062 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019891 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314064 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246515 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206777 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209719 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158123 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187159 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153785 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184355 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190256 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221551 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185076 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017512 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171111 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155355 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167035 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189552 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170775 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224493 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175835 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228241 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243756 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215015 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224366 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181101 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182096 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153232 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017028 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177301 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243525 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182627 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173843 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166799 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204297 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219431 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277543 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319397 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245433 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190134 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209304 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022179 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173609 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214717 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167301 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211243 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230751 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207936 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179225 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234939 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217768 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211508 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170624 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231541 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163345 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166943 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020712 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212041 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197554 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020555 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025173 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178255 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169561 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187619 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169063 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174397 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169396 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018318 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183434 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258089 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018205 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275958 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193575 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246022 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233323 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228582 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027369 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196465 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164404 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024523 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232351 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018319 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200928 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210282 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165794 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202357 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205989 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185048 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184171 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202403 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159427 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229686 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.017469 s, 3.0 GB/s , 0.0180181 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212358 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225198 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0184586 s, 2.8 GB/s , 0.017239 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019517 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213755 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176036 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168449 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167759 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169883 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169961 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01824 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200071 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207345 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170404 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019301 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258808 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421169 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238776 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377834 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382618 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178009 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173433 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218423 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025443 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180422 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181623 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200986 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022602 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153423 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171039 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201439 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176266 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147596 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202927 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172942 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161556 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233034 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182991 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01643 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208748 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180316 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157019 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186262 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263107 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281153 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328159 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245221 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329658 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215741 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258203 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233685 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274558 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210748 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240706 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227532 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236908 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287488 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278321 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345839 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272393 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483335 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304634 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565265 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468561 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179815 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180354 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193597 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249818 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293074 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029436 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292821 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492749 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270088 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311288 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213488 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235315 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246913 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236889 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267535 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270451 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226674 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238466 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289993 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279173 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422915 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334839 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357958 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192847 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023175 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250258 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250998 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335612 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054723 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026431 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288673 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362121 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263414 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313528 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254062 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296344 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226958 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217589 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192189 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195646 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229594 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232038 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0218167 s, 2.4 GB/s , 0.0218513 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227065 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253193 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0203521 s, 2.6 GB/s , 0.0220106 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212834 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221116 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268609 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251051 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237127 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198148 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211756 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216571 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210931 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340477 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337508 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192089 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029491 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025614 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280303 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204522 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321073 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217906 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350186 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203231 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210959 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231144 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237541 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235641 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220619 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194841 s, 2.7 GB/s , 0.0211107 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209798 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198909 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200329 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293637 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217154 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228608 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237569 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233132 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024606 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285964 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404135 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229603 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0234629 s, 2.2 GB/s , 0.0325745 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165204 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017609 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184893 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0207679 s, 2.5 GB/s , 0.0192395 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243159 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201973 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319465 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353223 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030235 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257193 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309845 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198415 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295771 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270835 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255106 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664585 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206897 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404967 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203203 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309857 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029569 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234557 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408737 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434506 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376178 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291401 s, 1.8 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0291565 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0292442 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336835 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209747 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208962 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306661 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327199 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272326 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234038 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271303 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204289 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244633 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241231 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425426 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250617 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023751 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020562 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225889 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243477 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337811 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399867 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317908 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266152 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274018 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342738 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240476 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258742 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247371 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223897 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271027 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296312 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246656 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234819 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212748 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215863 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261799 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207739 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316478 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228135 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317686 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235418 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266653 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207947 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250078 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379677 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349654 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309594 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236687 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261315 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264627 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03832 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240097 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025921 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210259 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200537 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226802 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222327 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310881 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230192 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234128 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240549 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264039 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284738 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224284 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022881 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225644 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02797 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267919 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234594 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0230529 s, 2.3 GB/s , 0.026675 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450418 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026153 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307755 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342112 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224093 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277354 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281769 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239668 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210497 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213071 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261478 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234958 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272955 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232759 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218652 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0230884 s, 2.3 GB/s , 0.0230979 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216139 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236792 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0215843 s, 2.4 GB/s , 0.021215 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172791 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201323 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197773 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283101 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250226 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230273 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246254 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230335 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0211904 s, 2.5 GB/s , 0.0207145 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210555 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213166 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188344 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194226 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0199025 s, 2.6 GB/s , 0.0199503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238672 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277332 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307193 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232405 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222375 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212666 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268483 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359123 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212598 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212354 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226882 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245745 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342416 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350582 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210266 s, 2.5 GB/s , 0.0210886 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250945 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241531 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281868 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222819 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229974 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259288 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303021 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200491 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282808 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317341 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352678 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291068 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449777 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025941 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236292 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309637 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416028 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324252 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410219 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243835 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208615 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026463 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244019 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032148 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219779 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230159 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284741 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282136 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203183 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250125 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220526 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273827 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241711 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274857 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259729 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345371 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326373 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373518 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259497 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264381 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498057 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510177 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375774 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326787 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027588 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260492 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256772 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241908 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260133 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233428 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242621 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217686 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341576 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562022 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319998 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439843 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489998 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441512 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278396 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025948 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275029 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225922 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285016 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240231 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281183 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262937 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339368 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384951 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380658 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272565 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341903 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401805 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327299 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342677 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283288 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300135 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294517 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239832 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251629 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321491 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304641 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240238 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345926 s, 1.5 GB/s , 0.0393568 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329084 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230716 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028224 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425163 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410833 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211124 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230685 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224786 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281344 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036089 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310763 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389014 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595007 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262059 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271907 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282543 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305727 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282067 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030467 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260984 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298504 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264572 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269589 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266178 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227131 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260138 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350498 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031123 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290841 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0291722 s, 1.8 GB/s , 0.0208715 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238426 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373026 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569078 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292516 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310138 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288161 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0489377 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.042123 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232418 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495948 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227258 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304777 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220624 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208781 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250732 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250251 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293682 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223763 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200818 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261272 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233986 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273413 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305797 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213989 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206636 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02039 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199681 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0211594 s, 2.5 GB/s , 0.0211702 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308316 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254785 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281623 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256655 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197834 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228424 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313247 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300724 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244366 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247561 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025965 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211917 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262537 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210114 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235281 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249198 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266586 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242903 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254362 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250055 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282218 s, 1.9 GB/s , 0.0283247 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295604 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301285 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246583 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238597 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225026 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239456 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204127 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207152 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324498 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0255445 s, 2.1 GB/s , 0.0362053 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259614 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204187 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204347 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021033 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299314 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0256548 s, 2.0 GB/s , 0.0214189 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027851 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022456 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289969 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255149 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330819 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274638 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299692 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218216 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281291 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338478 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530016 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234525 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027247 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294325 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360301 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480285 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193973 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237034 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028797 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295957 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252933 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354412 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209722 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240579 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349541 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284466 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616905 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231384 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252497 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365328 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313422 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362969 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023377 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287362 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0459614 s, 1.1 GB/s , 0.0366759 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485072 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325666 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259035 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202052 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277316 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258951 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307987 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272688 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0371503 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250276 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030239 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251564 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280618 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219159 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272878 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228028 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272885 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020674 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019293 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207863 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267794 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220519 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233984 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246939 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279376 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233272 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241814 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023089 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232726 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204405 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026024 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253127 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283824 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349846 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228352 s, 2.3 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222669 s, 2.4 GB/s , 0.0223852 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234794 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208128 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022562 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211678 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260329 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328752 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025436 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022584 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333888 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299607 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264186 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02323 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262575 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200593 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215002 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195052 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021019 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201795 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248343 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295148 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247675 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216505 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476758 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210928 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278093 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191489 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200794 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168409 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176767 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02702 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349948 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254943 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275357 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435568 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223969 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414282 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226424 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273511 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029376 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319457 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378908 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388298 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258435 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428125 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220895 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252853 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0269758 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0216364 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223733 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213374 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221209 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219922 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274939 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254603 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261324 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317373 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272427 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250435 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320215 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211592 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222523 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337872 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313798 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02483 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260174 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287204 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266731 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307712 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491613 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401149 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0631753 s, 830 MB/s , 0.030217 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425145 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393855 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024669 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216882 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226722 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225498 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269861 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307488 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297555 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243396 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255878 s, 2.0 GB/s , 0.0314411 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032431 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270533 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315085 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258943 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273605 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307233 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334663 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372202 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473989 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239982 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394122 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332214 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024586 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236117 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312336 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220198 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364015 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410645 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353896 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269035 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271216 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255527 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270879 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358731 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401669 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284618 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344606 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02933 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265894 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352608 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245612 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243125 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286954 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02933 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356352 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02837 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415394 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227335 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353807 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206839 s, 2.5 GB/s , 0.0216372 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025797 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379905 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440024 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292196 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023139 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231617 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271033 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249882 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275591 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214226 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190068 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237512 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250518 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236823 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230632 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267871 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024063 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254706 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188498 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208114 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297733 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245044 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020756 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245355 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254722 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299167 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221306 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243567 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222604 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286436 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373717 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223748 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252524 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219661 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259956 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220361 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0211646 s, 2.5 GB/s , 0.0213443 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023869 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230258 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224335 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237736 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247251 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021976 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230748 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025673 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194359 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232342 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238703 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227159 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204363 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241177 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266074 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215851 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328097 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350481 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220059 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261154 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244858 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0245773 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036096 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021064 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209141 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223003 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027508 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282707 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296362 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304997 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424458 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320189 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0541218 s, 969 MB/s , 0.0552924 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198599 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219499 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210101 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253795 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195254 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418255 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319431 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214877 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491139 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517579 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524674 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617673 s, 849 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369108 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321962 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455472 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243859 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285468 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205785 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214852 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223566 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228428 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230836 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294992 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351901 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286288 s, 1.8 GB/s , 0.0277147 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315438 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419903 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028448 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278958 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284067 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246851 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239011 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256681 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256354 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233663 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355787 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242734 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224036 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216414 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243269 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294332 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388004 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303202 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352166 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263492 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024073 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238722 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253764 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288234 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277019 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265724 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312791 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361031 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279608 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249871 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434863 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318815 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205222 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252612 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249998 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310467 s, 1.7 GB/s , 0.0265723 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263903 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244809 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215722 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216185 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233095 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268376 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261521 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383388 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393039 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031158 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438925 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394517 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041412 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231538 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327457 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200044 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215103 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208801 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201633 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316353 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325305 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315544 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367526 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276157 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268534 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0273664 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039237 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022384 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326411 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250789 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033377 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267879 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235312 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314271 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322068 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399254 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422734 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377643 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336705 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364234 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227116 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356207 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265199 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316185 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220863 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025659 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615963 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246557 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426273 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243903 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211013 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288472 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.023408 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028572 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396321 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438798 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032639 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404978 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411121 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053819 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285766 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218897 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030015 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353762 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268272 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346872 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346154 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361372 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430605 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402232 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024937 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425981 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205769 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02282 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302989 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238729 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306943 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269572 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0484683 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0439082 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348806 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034383 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335164 s, 1.6 GB/s Finish ======== /mnt/lustre/f44.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x1a4:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff88012b636000.stats= snapshot_time 1746141461.389491710 secs.nsecs start_time 1746141400.889582721 secs.nsecs elapsed_time 60.499908989 secs.nsecs llite.lustre-ffff88012e148000.stats= snapshot_time 1746141461.389961803 secs.nsecs start_time 1746141400.889614335 secs.nsecs elapsed_time 60.500347468 secs.nsecs read_bytes 1701 samples [bytes] 12288 1048576 1782579200 1869144299405312 read 159200 samples [usecs] 140 347111 109734049 1250026802477 ioctl 1 samples [reqs] open 3186 samples [usecs] 4 74109 3462893 61704318093 close 3186 samples [usecs] 4 61745 1856431 13642998059 seek 3183 samples [usecs] 2 979 22223 3349649 setattr 1 samples [usecs] 12727 12727 12727 161976529 getattr 17 samples [usecs] 111 2594 6270 7956062 mknod 1 samples [usecs] 6014 6014 6014 36168196 inode_permission 9840 samples [usecs] 1 21175 2538762 11445555884 opencount 3187 samples [reqs] 1 3186 5076892 10785008782 openclosetime 3185 samples [usecs] 0 370697 61878215 2651859131979 pcc_attach 1 samples [reqs] pcc_detach 264 samples [reqs] pcc_auto_attach 264 samples [reqs] pcc_hit_bytes 3150 samples [bytes] 52428800 52428800 165150720000 8658654068736000000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 attach 1 detach 264 auto_attach 264 llite.lustre-ffff88012b636000.pcc_async_threshold=1073741824 llite.lustre-ffff88012e148000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d44.sanity-pcc is idle now PASS 44 (66s) == sanity-pcc test 45: Concurrent read access from two mount points ========================================================== 19:17:44 (1746141464) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.144116 s, 728 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d45.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: 1b26a09f-0ab8-4d48-bfa4-f6f7f595a674 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1b26a09f-0ab8-4d48-bfa4-f6f7f595a674 (extents) (64bit) (large files) (huge files) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.184117 s, 570 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d45.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: 99f20e08-de14-44e9-a245-525a2e444035 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=99f20e08-de14-44e9-a245-525a2e444035 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d45.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} pcc: - pccpath: /mnt/pcc.d45.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff88012b636000.pcc_async_threshold=0 llite.lustre-ffff88012e148000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.42122 s, 124 MB/s /mnt/lustre/f45.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 421 0x1a5 0x240000400 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.566357 s, 92.6 MB/s 52428800 bytes (52 MB) copied, 0.535807 s, 97.9 MB/s , 0.540159 s, 97.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.514234 s, 102 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.545742 s, 96.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543049 s, 96.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.555063 s, 94.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.512358 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.526795 s, 99.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.571272 s, 91.8 MB/s 52428800 bytes (52 MB) copied, 0.580261 s, 90.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.53986 s, 97.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.550584 s, 95.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.54471 s, 96.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.549609 s, 95.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.547636 s, 95.7 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672461 s, 78.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.680717 s, 77.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.702872 s, 74.6 MB/s 52428800 bytes (52 MB) copied, 0.682744 s, 76.8 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.755555 s, 69.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.738 s, 71.0 MB/s 52428800 bytes (52 MB) copied, 0.680373 s, 77.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.758491 s, 69.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.680866 s, 77.0 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.684064 s, 76.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.691371 s, 75.8 MB/s 52428800 bytes (52 MB) copied, 0.706518 s, 74.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.7397 s, 70.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.692743 s, 75.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.771323 s, 68.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.737326 s, 71.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596199 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660064 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776422 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486309 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247752 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378736 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235462 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209316 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258792 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206836 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693326 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077309 s, 678 MB/s 52428800 bytes (52 MB) copied, 0.089453 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571883 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030712 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697294 s, 752 MB/s 52428800 bytes (52 MB) copied, 0.0863659 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502049 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797161 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290109 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585013 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221142 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501837 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510196 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437191 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342347 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297004 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276991 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543397 s, 965 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401004 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0380323 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548935 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757609 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631396 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632807 s, 829 MB/s , 0.0479448 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450996 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659147 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536941 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435635 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528028 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327265 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284108 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036052 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582338 s, 900 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942622 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105823 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672854 s, 779 MB/s 52428800 bytes (52 MB) copied, 0.103952 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783281 s, 669 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538996 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743319 s, 705 MB/s 52428800 bytes (52 MB) copied, 0.0893131 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126467 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431253 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0999745 s, 524 MB/s 52428800 bytes (52 MB) copied, 0.119146 s, 440 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047453 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0910445 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486858 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444202 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610849 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574498 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432096 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399436 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259669 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312824 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315204 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380938 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382626 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05095 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532791 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364196 s, 1.4 GB/s , 0.052461 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070736 s, 741 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907291 s, 578 MB/s 52428800 bytes (52 MB) copied, 0.0713871 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077108 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688714 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111294 s, 471 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797371 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620912 s, 844 MB/s 50+0 records in 50+0 records out , 0.133887 s, 392 MB/s 52428800 bytes (52 MB) copied, 0.08169 s, 642 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117526 s, 446 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0978417 s, 536 MB/s 52428800 bytes (52 MB) copied, 0.110949 s, 473 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0576325 s, 910 MB/s 52428800 bytes (52 MB) copied, 0.120734 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147427 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132207 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244577 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795064 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068644 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344098 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714207 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241676 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256661 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235683 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243601 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214621 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185961 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239695 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377812 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211707 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317584 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180306 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318484 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225948 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289665 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378906 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351989 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259614 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251066 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242838 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258505 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031639 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213072 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561191 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212451 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505941 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282411 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412647 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590405 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393361 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706993 s, 742 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0751269 s, 698 MB/s 52428800 bytes (52 MB) copied, 0.0622338 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627076 s, 836 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757506 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634034 s, 827 MB/s 52428800 bytes (52 MB) copied, 0.0561185 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527767 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290275 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447338 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020923 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230775 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183687 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259129 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298291 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402601 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425571 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0274659 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396067 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021971 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480263 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0471843 s, 1.1 GB/s , 0.0190768 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414531 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311331 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198708 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175957 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180845 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018566 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195342 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202206 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337899 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203287 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252252 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372871 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235612 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251105 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0178618 s, 2.9 GB/s , 0.0205087 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229108 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212435 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229591 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225429 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303251 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184198 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233744 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254789 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241085 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0343714 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254061 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231245 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184369 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189445 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345665 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505701 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593757 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501974 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693859 s, 756 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711886 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109286 s, 480 MB/s 52428800 bytes (52 MB) copied, 0.0596349 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445453 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030579 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729265 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0895112 s, 586 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680297 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708978 s, 739 MB/s 52428800 bytes (52 MB) copied, 0.0881924 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05969 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612072 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830345 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556396 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432274 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616395 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530315 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592825 s, 884 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238026 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0495176 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511056 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049514 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178063 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022109 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290935 s, 1.8 GB/s , 0.0257901 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220412 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203014 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018297 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213194 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209326 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232037 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199176 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223433 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030102 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341328 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489493 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488755 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591778 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352682 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481444 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506785 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040863 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0454833 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047076 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236493 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273305 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269662 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273716 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308027 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529975 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0408907 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0217898 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020578 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037923 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268286 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188889 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183052 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226215 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178064 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195244 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256383 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263604 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312169 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613433 s, 855 MB/s 52428800 bytes (52 MB) copied, 0.0581738 s, 901 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819848 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800349 s, 655 MB/s 52428800 bytes (52 MB) copied, 0.0840282 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100798 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119484 s, 439 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112521 s, 466 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119589 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110324 s, 475 MB/s 52428800 bytes (52 MB) copied, 0.108006 s, 485 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105735 s, 496 MB/s 52428800 bytes (52 MB) copied, 0.115311 s, 455 MB/s 52428800 bytes (52 MB) copied, 0.12513 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103677 s, 506 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811956 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901801 s, 581 MB/s 52428800 bytes (52 MB) copied, 0.0664715 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677117 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113417 s, 462 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05837 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343128 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775677 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056751 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026163 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216748 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245125 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220593 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278049 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314978 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695792 s, 754 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526466 s, 996 MB/s 52428800 bytes (52 MB) copied, 0.0705418 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762988 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505113 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434156 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0925181 s, 567 MB/s 52428800 bytes (52 MB) copied, 0.0639696 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099585 s, 526 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448504 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576329 s, 910 MB/s 52428800 bytes (52 MB) copied, 0.0675294 s, 776 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666442 s, 787 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810199 s, 647 MB/s 52428800 bytes (52 MB) copied, 0.0543885 s, 964 MB/s 52428800 bytes (52 MB) copied, 0.106481 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473818 s, 1.1 GB/s , 0.0530709 s, 988 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723835 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592003 s, 886 MB/s 52428800 bytes (52 MB) copied, 0.0598732 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320202 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704014 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282092 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231292 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183227 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018034 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237752 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190903 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245251 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02452 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302472 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202204 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243972 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03699 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293274 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411662 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724135 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032487 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385503 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0535732 s, 979 MB/s , 0.02188 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661851 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197145 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218681 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211303 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035982 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219896 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310404 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026083 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212559 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571568 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507538 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323579 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026138 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434323 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342741 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583155 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311301 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449024 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350036 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440076 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0388507 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347031 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457314 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376632 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267188 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021197 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245241 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170239 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219868 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233675 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168094 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273073 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241342 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285752 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238942 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435658 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0702108 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412826 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800118 s, 655 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695147 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848284 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.086617 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0956419 s, 548 MB/s 52428800 bytes (52 MB) copied, 0.0717498 s, 731 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0747604 s, 701 MB/s , 0.0627627 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053398 s, 982 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467229 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0539368 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383756 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385466 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369412 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225375 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279108 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191309 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218359 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189747 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241634 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270651 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302522 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303571 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198839 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193304 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402128 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362043 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258628 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195282 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300504 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216071 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191283 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230668 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208958 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402619 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352471 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294367 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204944 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262497 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324935 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207796 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200798 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386699 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327847 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221375 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218465 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289258 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017042 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185061 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201703 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186188 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272468 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190551 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0276513 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0410671 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266722 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311821 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352487 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0303392 s, 1.7 GB/s , 0.0244833 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230618 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237057 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220783 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165559 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255172 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206843 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206071 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328597 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018177 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196392 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194934 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262386 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334955 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206515 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432378 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569624 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344654 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370611 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356518 s, 1.5 GB/s , 0.0401906 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348388 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0325104 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541336 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455314 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509827 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564383 s, 929 MB/s 52428800 bytes (52 MB) copied, 0.0481414 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997753 s, 525 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10282 s, 510 MB/s 52428800 bytes (52 MB) copied, 0.0999639 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528361 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494764 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832286 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701747 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743188 s, 705 MB/s 52428800 bytes (52 MB) copied, 0.0507705 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227918 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543854 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732258 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322243 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262588 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031719 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203333 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483518 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385539 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294778 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224095 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0246258 s, 2.1 GB/s , 0.0286223 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0359294 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017878 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301574 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194025 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218873 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270688 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0252501 s, 2.1 GB/s , 0.028174 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426108 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309712 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427611 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325075 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0375267 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473495 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627694 s, 835 MB/s 52428800 bytes (52 MB) copied, 0.0548471 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502125 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0440711 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631203 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291144 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244548 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061786 s, 849 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561314 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327672 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204845 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229769 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231111 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293779 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195978 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197367 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165555 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362446 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020546 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235417 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232121 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212056 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219971 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194819 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183493 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211322 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214422 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159108 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238187 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347548 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036482 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370781 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278492 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0592808 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222256 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207972 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658442 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235181 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234663 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475946 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362226 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036328 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306975 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181254 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168764 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205044 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195479 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181493 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264818 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219363 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233619 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288913 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270607 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204428 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290346 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180156 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033963 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0197782 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0221445 s, 2.4 GB/s , 0.0333955 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196292 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022629 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189774 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188821 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206713 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186246 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179643 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167652 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155106 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182501 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169179 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170815 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172673 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188295 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193465 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203592 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189343 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0204077 s, 2.6 GB/s , 0.0176408 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245576 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254998 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238113 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325892 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0361573 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0278168 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415591 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0406921 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269546 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253768 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264145 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226968 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04156 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254004 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0189286 s, 2.8 GB/s , 0.0430226 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294064 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232018 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212156 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168607 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172718 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203605 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195098 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198356 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290632 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226515 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0406795 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041914 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021493 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214941 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283441 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204072 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228803 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191167 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021975 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225309 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204818 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240849 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165231 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229323 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179276 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016946 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212789 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170809 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188262 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209245 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016071 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251347 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215948 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243171 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199923 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283883 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264723 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317618 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213002 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180458 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037254 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438797 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458911 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275108 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194979 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444235 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194857 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241155 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185297 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197783 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184104 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196365 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017972 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175782 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390445 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337841 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433017 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351554 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0405949 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466823 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056537 s, 927 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275728 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0282582 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246411 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413817 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159602 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178078 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190142 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162889 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017301 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215557 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228826 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027764 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199056 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221462 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326831 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299647 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214749 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341225 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226996 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258994 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184252 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220381 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027743 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215679 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308782 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256709 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347536 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268642 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366747 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03425 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282702 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268381 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0620384 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394046 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245889 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414133 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243584 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347724 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280774 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0257311 s, 2.0 GB/s , 0.0348322 s, 1.5 GB/s , 0.0248731 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253697 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275673 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272527 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444226 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308335 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348021 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0453824 s, 1.2 GB/s , 0.0308004 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266307 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242439 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230564 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279888 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352976 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264524 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476558 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269475 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383169 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282878 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0633609 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453289 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315536 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242819 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495685 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542803 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364944 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593131 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031251 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551555 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245178 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385431 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302393 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02741 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306497 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025653 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306861 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290093 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448183 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789769 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391797 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374244 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.064826 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389765 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352094 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384475 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300421 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276848 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231555 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311554 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279379 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309445 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310817 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378325 s, 1.4 GB/s , 0.0383244 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061953 s, 846 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555269 s, 944 MB/s 52428800 bytes (52 MB) copied, 0.0575053 s, 912 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565542 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694824 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705527 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639505 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566702 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064804 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051544 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831939 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379931 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353743 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249943 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319779 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378237 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024386 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308522 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251887 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417781 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025129 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295278 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270401 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272056 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325397 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259091 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266321 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368858 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033502 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370908 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466264 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282302 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371805 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459163 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354055 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337848 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529445 s, 990 MB/s 52428800 bytes (52 MB) copied, 0.0707098 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690124 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690079 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544558 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387848 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055447 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328322 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489573 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451207 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654589 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306365 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357976 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311233 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262826 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384437 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043044 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447511 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367936 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035695 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303766 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0342164 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264213 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453469 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248778 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0593876 s, 883 MB/s , 0.0424185 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0522964 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224138 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251793 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275344 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267909 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232205 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239485 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022718 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323724 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443038 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449769 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0412078 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385879 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619801 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616327 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614309 s, 853 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549201 s, 955 MB/s 52428800 bytes (52 MB) copied, 0.0776184 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563675 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403138 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.06158 s, 851 MB/s , 0.0496288 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0432182 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696416 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660864 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573967 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534966 s, 980 MB/s , 0.0562007 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683074 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099826 s, 525 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731764 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492976 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909149 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102664 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771484 s, 680 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850206 s, 617 MB/s 52428800 bytes (52 MB) copied, 0.0773534 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481789 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392386 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499743 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959839 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437235 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832164 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352825 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265944 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029748 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023433 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237962 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205087 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225014 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221983 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0252753 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292471 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374361 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453571 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477541 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667446 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501251 s, 1.0 GB/s , 0.0862174 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051495 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741007 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716327 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656979 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306093 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723971 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0825782 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467599 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639572 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663333 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463551 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698359 s, 751 MB/s 52428800 bytes (52 MB) copied, 0.0587877 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798211 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651182 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623768 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0872434 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544892 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705577 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809626 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599688 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565258 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03236 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535813 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495457 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560945 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594568 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821782 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109219 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989018 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117653 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103969 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109198 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100251 s, 523 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105183 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0862232 s, 608 MB/s , 0.0752491 s, 697 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0790677 s, 663 MB/s , 0.0567364 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117635 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102571 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109037 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880405 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883248 s, 594 MB/s , 0.114862 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280011 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13019 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240407 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308967 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0364001 s, 1.4 GB/s , 0.0242769 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246348 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269821 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379886 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287494 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324566 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022406 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292647 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248145 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290312 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469913 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761553 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642248 s, 816 MB/s 52428800 bytes (52 MB) copied, 0.0795706 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105966 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11353 s, 462 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118655 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801554 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961825 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0945064 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108095 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818898 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879798 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125781 s, 417 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140106 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101615 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127483 s, 411 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116502 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155399 s, 337 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113589 s, 462 MB/s 52428800 bytes (52 MB) copied, 0.110804 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920705 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838854 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129098 s, 406 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0910432 s, 576 MB/s , 0.144425 s, 363 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142357 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107509 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933624 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910525 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510176 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538166 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046414 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287287 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229615 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368626 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268915 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360592 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261096 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346247 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319025 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347262 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353931 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560598 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450003 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442969 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326163 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393695 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558132 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0697563 s, 752 MB/s 52428800 bytes (52 MB) copied, 0.0707472 s, 741 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651809 s, 804 MB/s 52428800 bytes (52 MB) copied, 0.0795846 s, 659 MB/s 52428800 bytes (52 MB) copied, 0.098765 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0854981 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843863 s, 621 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103676 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985232 s, 532 MB/s 52428800 bytes (52 MB) copied, 0.112864 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0912535 s, 575 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946257 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564059 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535624 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556611 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505612 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934884 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810036 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315463 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466963 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547632 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479012 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527495 s, 994 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098221 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757677 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377474 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688588 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091924 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0859823 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055182 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772183 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0807162 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443377 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631012 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469257 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678469 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693578 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453375 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059174 s, 886 MB/s 52428800 bytes (52 MB) copied, 0.0779048 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550582 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539692 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460553 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549652 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850147 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387281 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418746 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319478 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301598 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355532 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243689 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232638 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208468 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241914 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267816 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391245 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562817 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270178 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327198 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544161 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312727 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653491 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517131 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475631 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402226 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366013 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405523 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289017 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564536 s, 929 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455732 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0791052 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539724 s, 971 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458675 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0682405 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607558 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0965314 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119053 s, 440 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780557 s, 672 MB/s 52428800 bytes (52 MB) copied, 0.130292 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116668 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137602 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123366 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992427 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13073 s, 401 MB/s , 0.0953192 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952888 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146083 s, 359 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933947 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103844 s, 505 MB/s 52428800 bytes (52 MB) copied, 0.134736 s, 389 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.126351 s, 415 MB/s , 0.0874381 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0841742 s, 623 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127615 s, 411 MB/s 52428800 bytes (52 MB) copied, 0.0741974 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830504 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0874707 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723681 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981049 s, 534 MB/s , 0.0945364 s, 555 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821047 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0887283 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601733 s, 871 MB/s 52428800 bytes (52 MB) copied, 0.107175 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605609 s, 866 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0583288 s, 899 MB/s , 0.0537739 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251539 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356341 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447173 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288777 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536343 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0916952 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403521 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847598 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0838917 s, 625 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.134105 s, 391 MB/s 52428800 bytes (52 MB) copied, 0.0882982 s, 594 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133382 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732581 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743893 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149722 s, 350 MB/s 52428800 bytes (52 MB) copied, 0.131177 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125389 s, 418 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100825 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.215261 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996809 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107284 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108576 s, 483 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383573 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601259 s, 872 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726379 s, 722 MB/s , 0.0787392 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0984425 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112025 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828017 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372044 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552187 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318541 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516302 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313688 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302706 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346561 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258639 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505742 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254564 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245905 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274913 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332245 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185969 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0197977 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253747 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229762 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400499 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498289 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624696 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683262 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640745 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635665 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771236 s, 680 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102484 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591333 s, 887 MB/s 52428800 bytes (52 MB) copied, 0.106397 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812148 s, 646 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971302 s, 540 MB/s 52428800 bytes (52 MB) copied, 0.119163 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954061 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.129415 s, 405 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959505 s, 546 MB/s 52428800 bytes (52 MB) copied, 0.118337 s, 443 MB/s 52428800 bytes (52 MB) copied, 0.0622251 s, 843 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873865 s, 600 MB/s 52428800 bytes (52 MB) copied, 0.0712673 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678496 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117411 s, 447 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638916 s, 821 MB/s 52428800 bytes (52 MB) copied, 0.0759676 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137734 s, 381 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592567 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479644 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.133569 s, 393 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432298 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.132852 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837409 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563914 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091768 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858806 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231764 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285272 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220295 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263219 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241876 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01945 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245508 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249432 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257542 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226148 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348875 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039104 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492279 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0545226 s, 962 MB/s , 0.0319822 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328581 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048647 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406154 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395358 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334093 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0507672 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572156 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641882 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700015 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660119 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891685 s, 588 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837611 s, 626 MB/s 52428800 bytes (52 MB) copied, 0.11577 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106172 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776392 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126197 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719271 s, 729 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119636 s, 438 MB/s 52428800 bytes (52 MB) copied, 0.120828 s, 434 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0998376 s, 525 MB/s , 0.117324 s, 447 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125389 s, 418 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13062 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12114 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114843 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534137 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994902 s, 527 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557353 s, 941 MB/s 52428800 bytes (52 MB) copied, 0.0771627 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112269 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455411 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818149 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572761 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493923 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0889117 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688545 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801134 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080225 s, 654 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114654 s, 457 MB/s 52428800 bytes (52 MB) copied, 0.0469594 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128761 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946626 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412487 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317374 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421547 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042134 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416415 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227734 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253857 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251441 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265214 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0324692 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376303 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333718 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295228 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449222 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457967 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410661 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346992 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628938 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523178 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740571 s, 708 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100788 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.0634135 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771675 s, 679 MB/s 52428800 bytes (52 MB) copied, 0.086594 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677949 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501416 s, 1.0 GB/s , 0.0598456 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102547 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673047 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100589 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828236 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731931 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654368 s, 801 MB/s , 0.0856859 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806345 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321039 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555013 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0939232 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864045 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269417 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380397 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310309 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293808 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796382 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0899539 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785104 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106167 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870699 s, 602 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0916006 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941317 s, 557 MB/s 52428800 bytes (52 MB) copied, 0.113463 s, 462 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0947522 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753058 s, 696 MB/s 52428800 bytes (52 MB) copied, 0.124162 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904434 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0951086 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636474 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139886 s, 375 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115702 s, 453 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107981 s, 486 MB/s 52428800 bytes (52 MB) copied, 0.067977 s, 771 MB/s 52428800 bytes (52 MB) copied, 0.0719709 s, 728 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727695 s, 720 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500094 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.065019 s, 806 MB/s 52428800 bytes (52 MB) copied, 0.0757483 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070663 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481069 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350884 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08336 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729438 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101877 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239483 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786443 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533618 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487808 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105642 s, 496 MB/s 52428800 bytes (52 MB) copied, 0.0663948 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892596 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410327 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840785 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205712 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259684 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370493 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343443 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287486 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360552 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026079 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324305 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256285 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340024 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250168 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270507 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378386 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338215 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513737 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612516 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758567 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622112 s, 843 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554887 s, 945 MB/s 52428800 bytes (52 MB) copied, 0.066069 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0744268 s, 704 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0931351 s, 563 MB/s , 0.0740848 s, 708 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738493 s, 710 MB/s 52428800 bytes (52 MB) copied, 0.0944479 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104835 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881266 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996176 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108674 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946453 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798476 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809913 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082443 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975984 s, 537 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07768 s, 675 MB/s 52428800 bytes (52 MB) copied, 0.0773523 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835958 s, 627 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731512 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402402 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0822515 s, 637 MB/s 52428800 bytes (52 MB) copied, 0.0645294 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442843 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610732 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052491 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419953 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037982 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402213 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321956 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438959 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342533 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299478 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384447 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323514 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493788 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341399 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341036 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257754 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276816 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422357 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287249 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422937 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263696 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294991 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253177 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293392 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461219 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0523536 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411145 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035631 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370871 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327799 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272975 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032116 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340601 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451436 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269937 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0224887 s, 2.3 GB/s , 0.0248124 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320996 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244897 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271909 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252024 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226052 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288731 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029193 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311656 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458848 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262685 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461403 s, 1.1 GB/s , 0.0495477 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295355 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406097 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284971 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535467 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549123 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345789 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0713375 s, 735 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629922 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432663 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626612 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465152 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533229 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056385 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376999 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030712 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0337516 s, 1.6 GB/s , 0.0278037 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417545 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027274 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259903 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334454 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241567 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338486 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300295 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340974 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0328461 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0240477 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269676 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416287 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.029363 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486938 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418621 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378254 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554342 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301511 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581606 s, 901 MB/s 52428800 bytes (52 MB) copied, 0.042851 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029082 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303028 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603397 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035174 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289634 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284348 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0236201 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0363975 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234666 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225129 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269951 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232264 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041967 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269282 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248747 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600676 s, 873 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515384 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0639635 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518665 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642822 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614968 s, 853 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875714 s, 599 MB/s 52428800 bytes (52 MB) copied, 0.0646442 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753601 s, 696 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520195 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079775 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496606 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0933602 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683468 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620543 s, 845 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376597 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.032183 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511599 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275541 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213602 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304819 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258103 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330943 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264865 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236649 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428641 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0630602 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647906 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561996 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519164 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562798 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575407 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612181 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452725 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313137 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513588 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456055 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278816 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397947 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024494 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294154 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279219 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274952 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285491 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321133 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409897 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274614 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449422 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433018 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362298 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284927 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407119 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046978 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031884 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278206 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028076 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025278 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0410033 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304503 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358253 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241401 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255925 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343203 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420377 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309767 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0560007 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417733 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355791 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311669 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261459 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024606 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307792 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0570388 s, 919 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.07049 s, 744 MB/s , 0.0649703 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051757 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686135 s, 764 MB/s 52428800 bytes (52 MB) copied, 0.0814138 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919025 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699813 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814521 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079804 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0784379 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595315 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612343 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0856571 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828899 s, 633 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798214 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074522 s, 704 MB/s 52428800 bytes (52 MB) copied, 0.0800833 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698624 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0808333 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652421 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585818 s, 895 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0977366 s, 536 MB/s 52428800 bytes (52 MB) copied, 0.0964056 s, 544 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490463 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654084 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055981 s, 937 MB/s 52428800 bytes (52 MB) copied, 0.0876488 s, 598 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0340575 s, 1.5 GB/s , 0.0468448 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395499 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273666 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247172 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236828 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241517 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245417 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455256 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501093 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700089 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633225 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578105 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685476 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527472 s, 994 MB/s , 0.0496668 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533882 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439645 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330488 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795342 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505338 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837938 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504903 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888539 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542348 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477553 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317399 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415358 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365738 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610345 s, 859 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605477 s, 866 MB/s 52428800 bytes (52 MB) copied, 0.0650731 s, 806 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717374 s, 731 MB/s 52428800 bytes (52 MB) copied, 0.083368 s, 629 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918995 s, 571 MB/s 52428800 bytes (52 MB) copied, 0.0812697 s, 645 MB/s 52428800 bytes (52 MB) copied, 0.0925318 s, 567 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875273 s, 599 MB/s 52428800 bytes (52 MB) copied, 0.0968858 s, 541 MB/s 52428800 bytes (52 MB) copied, 0.0835077 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0866536 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743557 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377207 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0733329 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092041 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507685 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520357 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473393 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223596 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222688 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242379 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276835 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279315 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232788 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269371 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339606 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349066 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286357 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408579 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530519 s, 988 MB/s , 0.034842 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522082 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483829 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382044 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392612 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588421 s, 891 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570766 s, 919 MB/s 52428800 bytes (52 MB) copied, 0.0575534 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686155 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498865 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649394 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558116 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06626 s, 791 MB/s , 0.040379 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531948 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321228 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467526 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264075 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252334 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320891 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269701 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331553 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245045 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239295 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250002 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192399 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231432 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261679 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283986 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320893 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215923 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233349 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241345 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349823 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313193 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540098 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620771 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703565 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0724275 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631715 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07983 s, 657 MB/s 52428800 bytes (52 MB) copied, 0.0755361 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619355 s, 847 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898615 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797958 s, 657 MB/s 52428800 bytes (52 MB) copied, 0.090783 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665198 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591282 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813781 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718487 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076283 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542719 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264614 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651962 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625229 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292531 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263508 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023088 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264557 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258307 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441656 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495606 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423255 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438642 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453975 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339127 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503596 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416969 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546756 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578587 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482472 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324824 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581745 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341937 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373017 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223227 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020035 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236611 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293692 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224991 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03302 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036039 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034464 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293828 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326975 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386138 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0307999 s, 1.7 GB/s , 0.0431028 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0581368 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359598 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048543 s, 1.1 GB/s , 0.038473 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042872 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548951 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362592 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465901 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414575 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555364 s, 944 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419014 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541153 s, 969 MB/s , 0.0641471 s, 817 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515035 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.055243 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542934 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460771 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399185 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357623 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668103 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033545 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655719 s, 800 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281864 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0631431 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301734 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271812 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247373 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239626 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251109 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209495 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228009 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225146 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257055 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176966 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250623 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202401 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020627 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211444 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220208 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224165 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263204 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471867 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254716 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294898 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532061 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268923 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304135 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572486 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041565 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383323 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0462181 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323871 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439025 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372787 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424958 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258015 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513901 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534447 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554644 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559058 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374264 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593562 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810989 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629508 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539875 s, 971 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624839 s, 839 MB/s 52428800 bytes (52 MB) copied, 0.0627728 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797345 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0585746 s, 895 MB/s 52428800 bytes (52 MB) copied, 0.057407 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065065 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462033 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610102 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302154 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029464 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187719 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212178 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235184 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243199 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304937 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023193 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027402 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233175 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311911 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430184 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02823 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575319 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514862 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355395 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410204 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264297 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305285 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691526 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293095 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253206 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262042 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319838 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449009 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394109 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075272 s, 697 MB/s 52428800 bytes (52 MB) copied, 0.0862123 s, 608 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753731 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058872 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082277 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827937 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892918 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881012 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791995 s, 662 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832904 s, 629 MB/s 52428800 bytes (52 MB) copied, 0.0812361 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818397 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723375 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734785 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700073 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550926 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683111 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737119 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471512 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345027 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540655 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413263 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685662 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297911 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492203 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269778 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063171 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268607 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280801 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235335 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328077 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493619 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752509 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783861 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886747 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102062 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907644 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100486 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0867317 s, 604 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0924067 s, 567 MB/s , 0.0932931 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971481 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704403 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110812 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898758 s, 583 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695035 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526615 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027703 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229642 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210679 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242446 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206826 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219517 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352446 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348305 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585177 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595752 s, 880 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603731 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640567 s, 818 MB/s 52428800 bytes (52 MB) copied, 0.0744449 s, 704 MB/s 52428800 bytes (52 MB) copied, 0.111667 s, 470 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10724 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11117 s, 472 MB/s 52428800 bytes (52 MB) copied, 0.0868547 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.101005 s, 519 MB/s 52428800 bytes (52 MB) copied, 0.0413098 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114192 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620884 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307684 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725447 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335797 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065367 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720003 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292708 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415727 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0277745 s, 1.9 GB/s , 0.0267807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259011 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305488 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275206 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266515 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263448 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209654 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201872 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254105 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262547 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472379 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286174 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438295 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316986 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341087 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427724 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429496 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485143 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611781 s, 857 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659586 s, 795 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677732 s, 774 MB/s 52428800 bytes (52 MB) copied, 0.0799093 s, 656 MB/s 52428800 bytes (52 MB) copied, 0.0704498 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0944205 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851439 s, 616 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496696 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971776 s, 540 MB/s 52428800 bytes (52 MB) copied, 0.0764419 s, 686 MB/s 52428800 bytes (52 MB) copied, 0.114292 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09879 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488401 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11421 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669076 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667112 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102367 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657614 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722691 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07371 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760518 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624291 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875898 s, 599 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766728 s, 684 MB/s 52428800 bytes (52 MB) copied, 0.0671765 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368769 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753389 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102809 s, 510 MB/s 52428800 bytes (52 MB) copied, 0.102347 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605331 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284438 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257234 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262214 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232892 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236863 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246365 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278537 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303482 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294356 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247686 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241598 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270411 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029007 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240605 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233285 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240228 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270828 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365077 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277741 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303878 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557893 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274639 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288818 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0866128 s, 605 MB/s 52428800 bytes (52 MB) copied, 0.0792058 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587255 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683948 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832303 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0752444 s, 697 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814021 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775508 s, 676 MB/s 52428800 bytes (52 MB) copied, 0.0760187 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817505 s, 641 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322473 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892482 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749522 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517964 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700331 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244572 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661828 s, 792 MB/s , 0.0370281 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440292 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284147 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355534 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351145 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296395 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347026 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283822 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358781 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529589 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489369 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491622 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0704331 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560702 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689939 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462864 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517123 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611433 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732571 s, 716 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722324 s, 726 MB/s 52428800 bytes (52 MB) copied, 0.0697826 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665243 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673324 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306523 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307755 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290723 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245795 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293682 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211434 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272301 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263143 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0284898 s, 1.8 GB/s , 0.0443327 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304095 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0393662 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321115 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.045015 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521787 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357958 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363708 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694332 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382853 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346812 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764858 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327081 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745248 s, 704 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666143 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438207 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478133 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0485252 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660074 s, 794 MB/s 52428800 bytes (52 MB) copied, 0.0929674 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070321 s, 746 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.115718 s, 453 MB/s 52428800 bytes (52 MB) copied, 0.101414 s, 517 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103064 s, 509 MB/s 52428800 bytes (52 MB) copied, 0.115754 s, 453 MB/s 52428800 bytes (52 MB) copied, 0.108561 s, 483 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106669 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750297 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766104 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811872 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133157 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931451 s, 563 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0954966 s, 549 MB/s 52428800 bytes (52 MB) copied, 0.108199 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487209 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031798 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454343 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241275 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419559 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341028 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389675 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221873 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255531 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248114 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274235 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265094 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329547 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197067 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238596 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207079 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189886 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02108 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526912 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386184 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658452 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688622 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820885 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824697 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104629 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09709 s, 540 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.107001 s, 490 MB/s , 0.1112 s, 471 MB/s 52428800 bytes (52 MB) copied, 0.0925099 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110687 s, 474 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134416 s, 390 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135286 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115264 s, 455 MB/s 52428800 bytes (52 MB) copied, 0.142097 s, 369 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144722 s, 362 MB/s 52428800 bytes (52 MB) copied, 0.123527 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919318 s, 570 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119074 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12094 s, 434 MB/s 52428800 bytes (52 MB) copied, 0.0900576 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102552 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507532 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106045 s, 494 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117822 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571288 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08296 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365381 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214129 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0265908 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.025987 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176736 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218685 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259812 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0324818 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0258762 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309482 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0412938 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224751 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271698 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236006 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226413 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230558 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249656 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277344 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275989 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277602 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294009 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336195 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517338 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363451 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322508 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380967 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361236 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022566 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184456 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186049 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285507 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415116 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505304 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636416 s, 824 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0826496 s, 634 MB/s , 0.0825217 s, 635 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0866392 s, 605 MB/s , 0.0786358 s, 667 MB/s 52428800 bytes (52 MB) copied, 0.0848913 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741826 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0993862 s, 528 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931326 s, 563 MB/s 52428800 bytes (52 MB) copied, 0.105651 s, 496 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10513 s, 499 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0823911 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108321 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763848 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686063 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743343 s, 705 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0485138 s, 1.1 GB/s , 0.0601477 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028437 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513537 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314746 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291813 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331318 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222432 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199183 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0194122 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0270629 s, 1.9 GB/s , 0.0255389 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264225 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280535 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271915 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332069 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311538 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298786 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378587 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0363603 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286035 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0419962 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0325529 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477725 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442815 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302595 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234741 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026289 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232389 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294686 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245966 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294622 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325493 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286487 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249134 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352239 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346595 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259313 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400113 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286912 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490333 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039213 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274209 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504508 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368822 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516701 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543017 s, 966 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692642 s, 757 MB/s 52428800 bytes (52 MB) copied, 0.0623382 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589812 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640123 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064207 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508158 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0712784 s, 736 MB/s , 0.059821 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769183 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541605 s, 968 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369328 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.061357 s, 854 MB/s , 0.063601 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419659 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559008 s, 938 MB/s 52428800 bytes (52 MB) copied, 0.0697634 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225799 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021353 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244571 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244738 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213711 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214713 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252115 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271279 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406749 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323723 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296404 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330295 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452059 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590741 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524701 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326012 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024456 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0366802 s, 1.4 GB/s , 0.0272297 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338666 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207611 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212466 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226507 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282432 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028431 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239618 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251899 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603247 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364046 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331857 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295076 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0337911 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216134 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0252256 s, 2.1 GB/s , 0.0348896 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304965 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325963 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362443 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239389 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227237 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218361 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023271 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243612 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251913 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289906 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0244244 s, 2.1 GB/s , 0.030665 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230146 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192479 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242757 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245329 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274128 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291412 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254715 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257436 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358456 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0331248 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272659 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358397 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392207 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235127 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024423 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328613 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209886 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247904 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271956 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0332454 s, 1.6 GB/s , 0.0318146 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405745 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190046 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193616 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196725 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210539 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216666 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235624 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335179 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373935 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0371337 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313036 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346111 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0400278 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401562 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540111 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509297 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342705 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518486 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398664 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315017 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255731 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228879 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213898 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205297 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260119 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268738 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223529 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240718 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225319 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232337 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228669 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223255 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273764 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248471 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221022 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226731 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581905 s, 901 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545382 s, 961 MB/s 52428800 bytes (52 MB) copied, 0.0656254 s, 799 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704113 s, 745 MB/s 52428800 bytes (52 MB) copied, 0.0716691 s, 732 MB/s 52428800 bytes (52 MB) copied, 0.0761973 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761864 s, 688 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828709 s, 633 MB/s 52428800 bytes (52 MB) copied, 0.0646855 s, 811 MB/s 52428800 bytes (52 MB) copied, 0.107481 s, 488 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678282 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880397 s, 596 MB/s 52428800 bytes (52 MB) copied, 0.102046 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116685 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789993 s, 664 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463547 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943833 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094672 s, 554 MB/s 52428800 bytes (52 MB) copied, 0.0673979 s, 778 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0552864 s, 948 MB/s , 0.0274136 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772478 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821298 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492603 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669768 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307871 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444444 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256625 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240028 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316938 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254616 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231702 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246574 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246431 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225728 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364351 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229791 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287367 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049709 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220416 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254733 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316093 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288061 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234718 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257191 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373221 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256684 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393585 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293187 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271085 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338582 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693216 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675556 s, 776 MB/s , 0.0469247 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565761 s, 927 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547887 s, 957 MB/s 52428800 bytes (52 MB) copied, 0.0550055 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119677 s, 438 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101306 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102713 s, 510 MB/s 52428800 bytes (52 MB) copied, 0.0823596 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0941203 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918535 s, 571 MB/s 52428800 bytes (52 MB) copied, 0.104885 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818853 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118877 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0784938 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110018 s, 477 MB/s , 0.107431 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165023 s, 318 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952174 s, 551 MB/s , 0.120925 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159518 s, 329 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851258 s, 616 MB/s 52428800 bytes (52 MB) copied, 0.130687 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136627 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15688 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069384 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799648 s, 656 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651107 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520167 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779351 s, 673 MB/s 52428800 bytes (52 MB) copied, 0.052833 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0908391 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810883 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658012 s, 797 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571639 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448661 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249441 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203941 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225115 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247594 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220284 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0267735 s, 2.0 GB/s , 0.0256606 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502536 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242827 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515684 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0344279 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260887 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207373 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438464 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356207 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185768 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277121 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206132 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178708 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238443 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239323 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361448 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026295 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269843 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314351 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045606 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605325 s, 866 MB/s 52428800 bytes (52 MB) copied, 0.0580424 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591779 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695142 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659315 s, 795 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10901 s, 481 MB/s 52428800 bytes (52 MB) copied, 0.0983675 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637154 s, 823 MB/s 52428800 bytes (52 MB) copied, 0.0842342 s, 622 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104699 s, 501 MB/s 52428800 bytes (52 MB) copied, 0.113501 s, 462 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637289 s, 823 MB/s 52428800 bytes (52 MB) copied, 0.120601 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353682 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.077724 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513717 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635233 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833878 s, 629 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588627 s, 891 MB/s 52428800 bytes (52 MB) copied, 0.0342122 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560633 s, 935 MB/s 52428800 bytes (52 MB) copied, 0.0484539 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033168 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0489809 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220943 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025921 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253935 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208522 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214514 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278088 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408244 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362109 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291759 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299094 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248779 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022714 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262211 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241009 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420846 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325818 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025345 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259242 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245834 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255881 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0299484 s, 1.8 GB/s , 0.0266877 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281126 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026902 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295104 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284807 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258479 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286629 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436152 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0324744 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315818 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513037 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408335 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031539 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706741 s, 742 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581244 s, 902 MB/s 52428800 bytes (52 MB) copied, 0.0422311 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538294 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258907 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0796058 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508398 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0713419 s, 735 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661172 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644501 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0782085 s, 670 MB/s 52428800 bytes (52 MB) copied, 0.0693585 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748661 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.086334 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745787 s, 703 MB/s , 0.0712149 s, 736 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0844779 s, 621 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868504 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758514 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847876 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732575 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729737 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074588 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711143 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462289 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535712 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498402 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497129 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426573 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021199 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210918 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321319 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227159 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188987 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313186 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0222297 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379142 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353897 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049706 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525242 s, 998 MB/s 52428800 bytes (52 MB) copied, 0.0702561 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650242 s, 806 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075437 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081118 s, 646 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0984875 s, 532 MB/s 52428800 bytes (52 MB) copied, 0.101267 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933097 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102488 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982474 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0852278 s, 615 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833951 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106643 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777419 s, 674 MB/s 52428800 bytes (52 MB) copied, 0.0814942 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109113 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578963 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735425 s, 713 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465582 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424907 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437002 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438895 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463927 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505747 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534279 s, 981 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346378 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442326 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517048 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468306 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316334 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339208 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233537 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0306982 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297257 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274084 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347597 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424013 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316191 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.036399 s, 1.4 GB/s , 0.0433732 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360351 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448254 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287623 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031495 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041164 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363251 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183175 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227108 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030188 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027752 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465474 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385672 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346826 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030071 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311219 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032298 s, 1.6 GB/s , 0.0343405 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239664 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220366 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206958 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024909 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227553 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304375 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468871 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231285 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257421 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228746 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236758 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223374 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024194 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310122 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268741 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318852 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485298 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03313 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402926 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294449 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044294 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307249 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448857 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216224 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195744 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0191848 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163998 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023292 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338943 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285941 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309581 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216501 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251431 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289652 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0450712 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056928 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044458 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504479 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0519048 s, 1.0 GB/s , 0.0489168 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423202 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640611 s, 818 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611815 s, 857 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0987914 s, 531 MB/s 52428800 bytes (52 MB) copied, 0.0717048 s, 731 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675008 s, 777 MB/s 52428800 bytes (52 MB) copied, 0.0737522 s, 711 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858444 s, 611 MB/s 52428800 bytes (52 MB) copied, 0.100491 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538273 s, 974 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0674329 s, 777 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450621 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793204 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029341 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732857 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308313 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231796 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252105 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252688 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348924 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309256 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562661 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717098 s, 731 MB/s , 0.0699286 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692142 s, 757 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881512 s, 595 MB/s 52428800 bytes (52 MB) copied, 0.0847336 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0857665 s, 611 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091871 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846301 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688406 s, 762 MB/s 52428800 bytes (52 MB) copied, 0.0853727 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044827 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801195 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506039 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0801928 s, 654 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0235236 s, 2.2 GB/s , 0.0236298 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325161 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200384 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254239 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291192 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301496 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425664 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243685 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354314 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329032 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317037 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268976 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263194 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036749 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0419474 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428265 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027677 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470402 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432914 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317786 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261235 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269163 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229509 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231777 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379906 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322066 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393519 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469534 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343739 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250173 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486798 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381858 s, 1.4 GB/s , 0.0410457 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408606 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592565 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599981 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554184 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802153 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614396 s, 853 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440741 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579676 s, 904 MB/s 52428800 bytes (52 MB) copied, 0.0523166 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04962 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315523 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425415 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227086 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293954 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242216 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254638 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256713 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341644 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320272 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383933 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542478 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035741 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522198 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333318 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237182 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228464 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287372 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282015 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024807 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359329 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277794 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487027 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599364 s, 875 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775009 s, 676 MB/s 52428800 bytes (52 MB) copied, 0.0533759 s, 982 MB/s 52428800 bytes (52 MB) copied, 0.0521233 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530768 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543806 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0945367 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0867332 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835104 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791809 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069999 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793311 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754616 s, 695 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800788 s, 655 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.125493 s, 418 MB/s , 0.0661345 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0945647 s, 554 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102793 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0905826 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678894 s, 772 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436058 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584617 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out , 0.071952 s, 729 MB/s 52428800 bytes (52 MB) copied, 0.048029 s, 1.1 GB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0849385 s, 617 MB/s , 0.0745139 s, 704 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642648 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223869 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256055 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0243102 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191785 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023162 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198712 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249389 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394598 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441528 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576585 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337057 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417608 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621629 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470036 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688487 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753611 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704201 s, 745 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629457 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720347 s, 728 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639019 s, 820 MB/s 52428800 bytes (52 MB) copied, 0.0704786 s, 744 MB/s 52428800 bytes (52 MB) copied, 0.0861059 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811802 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0935438 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0859193 s, 610 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850806 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0968675 s, 541 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0898167 s, 584 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111879 s, 469 MB/s 52428800 bytes (52 MB) copied, 0.0937133 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761334 s, 689 MB/s 52428800 bytes (52 MB) copied, 0.119648 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568009 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980466 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0856857 s, 612 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403388 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0749623 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049983 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423499 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386242 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332662 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309346 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509986 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485207 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226273 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210975 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254537 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246094 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243416 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022731 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197521 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299646 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040792 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465517 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302467 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046164 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0649246 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555968 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423707 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347268 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646048 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457228 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268918 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237641 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240416 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346826 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252749 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394537 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535774 s, 979 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0732912 s, 715 MB/s , 0.0708613 s, 740 MB/s 52428800 bytes (52 MB) copied, 0.0651147 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781451 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755489 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694389 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837661 s, 626 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0649856 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876696 s, 598 MB/s , 0.0761345 s, 689 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879213 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581419 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066516 s, 788 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581767 s, 901 MB/s 52428800 bytes (52 MB) copied, 0.0581314 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438305 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380233 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353601 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318846 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406474 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323987 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229345 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225821 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01765 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163037 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220408 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271611 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265773 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389319 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401473 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423741 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537515 s, 975 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404844 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0368386 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039009 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452229 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370193 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215991 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207489 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257883 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317163 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039405 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283711 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559543 s, 937 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452337 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506337 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671954 s, 780 MB/s 52428800 bytes (52 MB) copied, 0.0647983 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055447 s, 946 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692385 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047617 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465518 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777252 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541233 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328813 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742155 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850582 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719711 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049354 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604076 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381475 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567777 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410589 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504011 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040285 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229947 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400158 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270835 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211196 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021908 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268402 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031483 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242103 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224984 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299693 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314712 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439649 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400813 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398863 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439101 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410953 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343865 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0294142 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236969 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242059 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019924 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016917 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229258 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330855 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277983 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027729 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376158 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.028683 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0449167 s, 1.2 GB/s , 0.059886 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610843 s, 858 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.082383 s, 636 MB/s 52428800 bytes (52 MB) copied, 0.062892 s, 834 MB/s , 0.0662657 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472279 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059109 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470905 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933339 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.097227 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725813 s, 722 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602223 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334348 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201197 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248976 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275498 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285225 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436651 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294497 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0524196 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397336 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498871 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488007 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493693 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308934 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335808 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0236753 s, 2.2 GB/s , 0.0359427 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237168 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230176 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339995 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024156 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207944 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240508 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267654 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236967 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0208713 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.022025 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209473 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215546 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214847 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234254 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320985 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277035 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235189 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470322 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0527799 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415924 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364436 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271438 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427577 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327735 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327891 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519959 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599097 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109615 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717801 s, 730 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824276 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959799 s, 546 MB/s 52428800 bytes (52 MB) copied, 0.0986622 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0837656 s, 626 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980701 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958223 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623733 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775094 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10366 s, 506 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108215 s, 484 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937966 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0989693 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13728 s, 382 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584496 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576421 s, 910 MB/s 52428800 bytes (52 MB) copied, 0.0950063 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693724 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146053 s, 359 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0856862 s, 612 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121004 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106837 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717771 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056378 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0799489 s, 656 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.097792 s, 536 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574699 s, 912 MB/s 52428800 bytes (52 MB) copied, 0.0767274 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659522 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386133 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561148 s, 934 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333314 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0329089 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232292 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226845 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231412 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264613 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234948 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281866 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202557 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326438 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222101 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0313575 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330989 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758965 s, 691 MB/s 52428800 bytes (52 MB) copied, 0.0674115 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494966 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0577531 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415406 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830603 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511197 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09388 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464215 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483821 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384081 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.087491 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504905 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322727 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045257 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189197 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250762 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022817 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209876 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246116 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520592 s, 1.0 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0330566 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0477899 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072151 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476561 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694218 s, 755 MB/s 52428800 bytes (52 MB) copied, 0.0843367 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0896738 s, 585 MB/s 52428800 bytes (52 MB) copied, 0.0811173 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485419 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600316 s, 873 MB/s 52428800 bytes (52 MB) copied, 0.109695 s, 478 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748001 s, 701 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704963 s, 744 MB/s 52428800 bytes (52 MB) copied, 0.050182 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471591 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424584 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141856 s, 370 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660624 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500185 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560447 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527345 s, 994 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447653 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503205 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703121 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846232 s, 620 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551782 s, 950 MB/s , 0.0552702 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271253 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.067847 s, 773 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333588 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255762 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273111 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286579 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210191 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297432 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194396 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192981 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275072 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388893 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248801 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397009 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324018 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232918 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339878 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260449 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021775 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251879 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022328 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300564 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389458 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424307 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575513 s, 911 MB/s 52428800 bytes (52 MB) copied, 0.0503778 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621515 s, 844 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643967 s, 814 MB/s 52428800 bytes (52 MB) copied, 0.0720076 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826199 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520761 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616567 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618266 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592639 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055501 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565086 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025788 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543595 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037119 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509726 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304653 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522383 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484502 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570061 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615818 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697684 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697211 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728425 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620454 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738292 s, 710 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677561 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383806 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465469 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421088 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025394 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338656 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326944 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262251 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204467 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255528 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300959 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241927 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288571 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243517 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266601 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280126 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253504 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259531 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178436 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256275 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026648 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189508 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239898 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208247 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215195 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283623 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025923 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231134 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274951 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272651 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022924 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036227 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244354 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296944 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355076 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283274 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250694 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283376 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321194 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518688 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565615 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338499 s, 1.5 GB/s , 0.0457614 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374941 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045599 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788857 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242835 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628063 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624832 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474839 s, 1.1 GB/s , 0.0344516 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050346 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418321 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021738 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210063 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241956 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308714 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313513 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296277 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373965 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266724 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290761 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509958 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512784 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297152 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309933 s, 1.7 GB/s , 0.0391533 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346083 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293324 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032952 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324062 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227536 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417091 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279559 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507784 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293769 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043642 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255954 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486613 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506287 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0499929 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519787 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567791 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637494 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477274 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721719 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059025 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508739 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368426 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070049 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230731 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593625 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749968 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107457 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104261 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846596 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127968 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115162 s, 455 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786245 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124465 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750076 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13767 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141323 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170382 s, 308 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137954 s, 380 MB/s 52428800 bytes (52 MB) copied, 0.15622 s, 336 MB/s 52428800 bytes (52 MB) copied, 0.130177 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14248 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.170329 s, 308 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155336 s, 338 MB/s , 0.163683 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919532 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140839 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113125 s, 463 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933396 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075925 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127783 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106517 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492717 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111215 s, 471 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0960035 s, 546 MB/s 52428800 bytes (52 MB) copied, 0.0911316 s, 575 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692317 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454866 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623039 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616926 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253759 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252741 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217789 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258865 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209674 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370207 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274224 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369311 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350607 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274767 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02237 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288943 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268539 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250812 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336432 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336291 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060992 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549456 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624316 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341836 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032148 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357818 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0722929 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631229 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340153 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412169 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349402 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301249 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415362 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318184 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.052235 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756905 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075841 s, 691 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0902866 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0959111 s, 547 MB/s 52428800 bytes (52 MB) copied, 0.0830725 s, 631 MB/s 52428800 bytes (52 MB) copied, 0.0825472 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112136 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827453 s, 634 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0982831 s, 533 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.100817 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.115639 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114508 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976482 s, 537 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.09494 s, 552 MB/s 52428800 bytes (52 MB) copied, 0.0705081 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780255 s, 672 MB/s 52428800 bytes (52 MB) copied, 0.0566315 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038561 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376309 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372132 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250837 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0278083 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230985 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231594 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251326 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307334 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262769 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291919 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214681 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481806 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306051 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616521 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412217 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342857 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493411 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033194 s, 1.6 GB/s , 0.037087 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036861 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312819 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357047 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369976 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379337 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520352 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0933093 s, 562 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600802 s, 873 MB/s 52428800 bytes (52 MB) copied, 0.085355 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833748 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638743 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113821 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102693 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795866 s, 659 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819717 s, 640 MB/s 52428800 bytes (52 MB) copied, 0.0727645 s, 721 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581714 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123442 s, 425 MB/s 52428800 bytes (52 MB) copied, 0.0895008 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675033 s, 777 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948783 s, 553 MB/s 52428800 bytes (52 MB) copied, 0.0743407 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668321 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626897 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685889 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101393 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154047 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573228 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561738 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139115 s, 377 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722533 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0520656 s, 1.0 GB/s , 0.117946 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100054 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263234 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472836 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0511239 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042164 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306606 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205547 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209358 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277774 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379878 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228696 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305286 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302986 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213447 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256907 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352429 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297858 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411057 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027866 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275449 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218839 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367954 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264005 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280594 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343246 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257459 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322503 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223602 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262032 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021206 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256646 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211352 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302758 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0569354 s, 921 MB/s 52428800 bytes (52 MB) copied, 0.056015 s, 936 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368335 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.041948 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358187 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334571 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028256 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344111 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367276 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296062 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639052 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281928 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349919 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240453 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233035 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411538 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248052 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251048 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252956 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291787 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316274 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341798 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187361 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549994 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451769 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687306 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805679 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678458 s, 773 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931953 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819425 s, 640 MB/s 52428800 bytes (52 MB) copied, 0.0870519 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605686 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074026 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757819 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0903773 s, 580 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635837 s, 825 MB/s 52428800 bytes (52 MB) copied, 0.0611198 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103353 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724827 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100343 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491036 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683367 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462794 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0848164 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384329 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226598 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301566 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342256 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443561 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480083 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481892 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595168 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362862 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456848 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791261 s, 663 MB/s 52428800 bytes (52 MB) copied, 0.0597564 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478821 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688177 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443112 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0580056 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606652 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479775 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701709 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616602 s, 850 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587072 s, 893 MB/s 52428800 bytes (52 MB) copied, 0.0612773 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793819 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776972 s, 675 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655215 s, 800 MB/s 52428800 bytes (52 MB) copied, 0.0549311 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347424 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662369 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455797 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717809 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261343 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250803 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404101 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227605 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252243 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205202 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540089 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239141 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301598 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412582 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183245 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231562 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266296 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339364 s, 1.5 GB/s , 0.0410524 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026838 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048084 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0543371 s, 965 MB/s 52428800 bytes (52 MB) copied, 0.0609581 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307969 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607397 s, 863 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054222 s, 967 MB/s 52428800 bytes (52 MB) copied, 0.0496452 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037413 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398242 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770483 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340193 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478177 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584272 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042429 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467761 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599918 s, 874 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546478 s, 959 MB/s 52428800 bytes (52 MB) copied, 0.0614614 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509235 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036488 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494705 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449102 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438734 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303819 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212926 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211704 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227646 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026725 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294577 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251415 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796032 s, 659 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709178 s, 739 MB/s 52428800 bytes (52 MB) copied, 0.0791428 s, 662 MB/s 52428800 bytes (52 MB) copied, 0.064764 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677103 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704856 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710531 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639117 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0951205 s, 551 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.086611 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0717132 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569489 s, 921 MB/s 52428800 bytes (52 MB) copied, 0.0681102 s, 770 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159234 s, 329 MB/s 52428800 bytes (52 MB) copied, 0.0871454 s, 602 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150751 s, 348 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112014 s, 468 MB/s 52428800 bytes (52 MB) copied, 0.0852772 s, 615 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689936 s, 760 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719296 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116949 s, 448 MB/s 52428800 bytes (52 MB) copied, 0.0602378 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530844 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.097356 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371123 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398665 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0389817 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0208179 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949218 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753171 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178183 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290418 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258844 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211379 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327993 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274457 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255425 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252218 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384214 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327359 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273579 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504772 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533018 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409446 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276755 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0356937 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024536 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238537 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441141 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432489 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207984 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249985 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408743 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023132 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417674 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312035 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636797 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026629 s, 2.0 GB/s , 0.0411203 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448264 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455058 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546308 s, 960 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612891 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414162 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0537914 s, 975 MB/s 52428800 bytes (52 MB) copied, 0.0742663 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607224 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620413 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426601 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0582452 s, 900 MB/s , 0.0821819 s, 638 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582344 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222957 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358595 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291834 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233464 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023732 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293907 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243262 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355677 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190886 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338374 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216005 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255751 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270993 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265061 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354366 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408695 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045592 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0552265 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045101 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450123 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415513 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.074225 s, 706 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870193 s, 602 MB/s 52428800 bytes (52 MB) copied, 0.0808487 s, 648 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0963433 s, 544 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687355 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748354 s, 701 MB/s 52428800 bytes (52 MB) copied, 0.0627519 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642081 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118856 s, 441 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708977 s, 739 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910019 s, 576 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069703 s, 752 MB/s 52428800 bytes (52 MB) copied, 0.0572008 s, 917 MB/s 52428800 bytes (52 MB) copied, 0.0531329 s, 987 MB/s 52428800 bytes (52 MB) copied, 0.087267 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386325 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299148 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528408 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234217 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445195 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199219 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239601 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294188 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0356791 s, 1.5 GB/s , 0.0354143 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02989 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411937 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027446 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290992 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391593 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229765 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345664 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420881 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0784709 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324563 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748123 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510623 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404649 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949413 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676485 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517858 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478786 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0917278 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0971307 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111622 s, 470 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946896 s, 554 MB/s 52428800 bytes (52 MB) copied, 0.104456 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403751 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608102 s, 862 MB/s 52428800 bytes (52 MB) copied, 0.0773934 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104411 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377056 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109855 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456458 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103945 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412253 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802026 s, 654 MB/s 52428800 bytes (52 MB) copied, 0.0607135 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664953 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615307 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123391 s, 425 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558642 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724699 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724225 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063743 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0733539 s, 715 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863323 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426882 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620549 s, 845 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550022 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108705 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102381 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610013 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571395 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249146 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379176 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310272 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201633 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236648 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444829 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352016 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482067 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363674 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603449 s, 869 MB/s 52428800 bytes (52 MB) copied, 0.0479504 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491418 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535582 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338097 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326936 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714489 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247083 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428448 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596122 s, 879 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891232 s, 588 MB/s 52428800 bytes (52 MB) copied, 0.0697875 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573974 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0615499 s, 852 MB/s 52428800 bytes (52 MB) copied, 0.071974 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0896583 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070841 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706269 s, 742 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646493 s, 811 MB/s 52428800 bytes (52 MB) copied, 0.103928 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485974 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124469 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114484 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446558 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168672 s, 311 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397848 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103093 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131015 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178556 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388843 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329655 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358706 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306736 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0391357 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319276 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840113 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360884 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367147 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0824478 s, 636 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624755 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757985 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504902 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074141 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729675 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453396 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556083 s, 943 MB/s 52428800 bytes (52 MB) copied, 0.0560397 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753431 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628468 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586732 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357212 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486091 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420532 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476316 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534085 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316872 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027778 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0747267 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206546 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211627 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247292 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384936 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369043 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312514 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240101 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251578 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195477 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271632 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265608 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460518 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449765 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385688 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589566 s, 889 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783125 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482281 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11007 s, 476 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641986 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111693 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147399 s, 356 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121556 s, 431 MB/s 52428800 bytes (52 MB) copied, 0.0993906 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135409 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119018 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112189 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120611 s, 435 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137455 s, 381 MB/s 52428800 bytes (52 MB) copied, 0.154174 s, 340 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126027 s, 416 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14152 s, 370 MB/s 52428800 bytes (52 MB) copied, 0.152347 s, 344 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.11144 s, 470 MB/s , 0.132545 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105518 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996243 s, 526 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699273 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11155 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142674 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110755 s, 473 MB/s 52428800 bytes (52 MB) copied, 0.115065 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112557 s, 466 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129676 s, 404 MB/s 52428800 bytes (52 MB) copied, 0.0725006 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664234 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403153 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498905 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0526968 s, 995 MB/s , 0.0601659 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264867 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268128 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272422 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384498 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258246 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327467 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364364 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289543 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348751 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381685 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375751 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362879 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314328 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528645 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367534 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429322 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348289 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0756566 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478753 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542854 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330211 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647391 s, 810 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459507 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600083 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478572 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451835 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107239 s, 489 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102974 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039739 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124895 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414404 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107047 s, 490 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048858 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124333 s, 422 MB/s 52428800 bytes (52 MB) copied, 0.0832258 s, 630 MB/s 52428800 bytes (52 MB) copied, 0.126477 s, 415 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127194 s, 412 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816732 s, 642 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.174631 s, 300 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885792 s, 592 MB/s 52428800 bytes (52 MB) copied, 0.0869343 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0988854 s, 530 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0897639 s, 584 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0874335 s, 600 MB/s , 0.0681279 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532408 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312347 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153024 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937348 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196865 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346205 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278178 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218939 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202835 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370899 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251513 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332425 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030946 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236962 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486591 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408145 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469808 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0616149 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.091783 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636417 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475221 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0605203 s, 866 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0944635 s, 555 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0210347 s, 2.5 GB/s , 0.0218011 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161517 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171297 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226941 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244446 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393071 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210357 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244302 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388205 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343305 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421954 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193531 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341383 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449939 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561268 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611956 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526418 s, 996 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680181 s, 771 MB/s 52428800 bytes (52 MB) copied, 0.0767303 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636651 s, 824 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609859 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772393 s, 679 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613518 s, 855 MB/s 52428800 bytes (52 MB) copied, 0.094097 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430108 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301913 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222504 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391109 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226925 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311877 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427096 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275321 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404021 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358968 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544333 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382908 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417575 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230594 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342832 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597661 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542722 s, 966 MB/s , 0.0364977 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319706 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376414 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238485 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299992 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177323 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207484 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203696 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0207452 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029015 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186051 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204199 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194787 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208771 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308727 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491673 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459188 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0591376 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102841 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438091 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797439 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044757 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08417 s, 623 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929241 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789041 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771588 s, 679 MB/s 52428800 bytes (52 MB) copied, 0.0714199 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416348 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682555 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601502 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315806 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438442 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306547 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413156 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228015 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300277 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018543 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305936 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216602 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246569 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244155 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044487 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416353 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318476 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426949 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332586 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199034 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299063 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240902 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362909 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273554 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229184 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200507 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268858 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202195 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189086 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236213 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214623 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196045 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350416 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191795 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391757 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239295 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267542 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411373 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278409 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302424 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384073 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224628 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230526 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243391 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418805 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377965 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236961 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329856 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225381 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241043 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240667 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215264 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388953 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266393 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298228 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178867 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240162 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018731 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224574 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398503 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426361 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456247 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516563 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0825609 s, 635 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672429 s, 780 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771345 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0686328 s, 764 MB/s , 0.0687511 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0739243 s, 709 MB/s 52428800 bytes (52 MB) copied, 0.0740674 s, 708 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123601 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059539 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077272 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365486 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587127 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500814 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511391 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644584 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623668 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627663 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434823 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221643 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257191 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594126 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771914 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707422 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535282 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032395 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439046 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0723473 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454513 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541978 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227527 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540621 s, 970 MB/s 52428800 bytes (52 MB) copied, 0.0667972 s, 785 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028216 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723444 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827079 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0951319 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575643 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251892 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183143 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171414 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179362 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224871 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182578 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221675 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195757 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283786 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240519 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272169 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030852 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304932 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215206 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342303 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249941 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021242 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293953 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214477 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277589 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336844 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240132 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338059 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339609 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333404 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164648 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202181 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261069 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165665 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173266 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187171 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258332 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259788 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316643 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218209 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0650883 s, 806 MB/s 52428800 bytes (52 MB) copied, 0.0297264 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382981 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264102 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380317 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395601 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0351619 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330935 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567532 s, 924 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603622 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461266 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501753 s, 1.0 GB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0427065 s, 1.2 GB/s , 0.0572194 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540819 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052608 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231594 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381958 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0995185 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.082885 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059097 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351419 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602911 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447066 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332903 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223112 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252858 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249752 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240529 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0237216 s, 2.2 GB/s , 0.027405 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214763 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192454 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147685 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159401 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161424 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174141 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02049 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172636 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174831 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023242 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222881 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02246 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171365 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019698 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231172 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225965 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187657 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016483 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0212704 s, 2.5 GB/s , 0.0175274 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207699 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212203 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256689 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158626 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173557 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163563 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185575 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253122 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205269 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204231 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183396 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198569 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167024 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195843 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186135 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261428 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213655 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162042 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230543 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180168 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221045 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218257 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0313307 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246594 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029658 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0540638 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308936 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275858 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283286 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397161 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224144 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192765 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219764 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186375 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197162 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168168 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168975 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199914 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218026 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235837 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02983 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201119 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202156 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321106 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263619 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306199 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281468 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512889 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313132 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201422 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181622 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161364 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172055 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176019 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195474 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236694 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308791 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192665 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023832 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170581 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267342 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204604 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241064 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273257 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0177674 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161288 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175575 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211472 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168016 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218438 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02024 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189113 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019253 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318588 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190948 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301078 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240411 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182015 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183397 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165305 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166765 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220555 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198186 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195884 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272513 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169199 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164523 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01646 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194057 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0181771 s, 2.9 GB/s , 0.0182436 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018503 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180166 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333171 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023922 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325664 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350576 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020314 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298485 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179909 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027286 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188354 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197644 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233025 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194648 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194743 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258334 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208622 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031258 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183374 s, 2.9 GB/s , 0.0337902 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291154 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286027 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210336 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220868 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.015864 s, 3.3 GB/s , 0.0153702 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171259 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232642 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223379 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185571 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185367 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200978 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169089 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165132 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200611 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205225 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017515 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238311 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248771 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020678 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195521 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180952 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257046 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172359 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160603 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253493 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230337 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331824 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416324 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255638 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026144 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027804 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245366 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223997 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289238 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192511 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196603 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258306 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218331 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186435 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173376 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293915 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01727 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152532 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017252 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207342 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020135 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309493 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167606 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018005 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161908 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156706 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016184 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0217751 s, 2.4 GB/s , 0.0218262 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179837 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179681 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182341 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243548 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212815 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176304 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230098 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251706 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243305 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194759 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0314187 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249339 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224404 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284711 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296256 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257556 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265259 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265382 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185017 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183952 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166157 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218793 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189801 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160803 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225132 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206465 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209627 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187525 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215476 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0328818 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210939 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169187 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219842 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018433 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250047 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175921 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198354 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233418 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194342 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332267 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184376 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240242 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219095 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189889 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205778 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205695 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204197 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272684 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182349 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020642 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262332 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182096 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384436 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234189 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358467 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477668 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470547 s, 1.1 GB/s , 0.0353314 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553885 s, 947 MB/s 52428800 bytes (52 MB) copied, 0.0475371 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379448 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382253 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379989 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335417 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0295433 s, 1.8 GB/s , 0.0248665 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395989 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187646 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174594 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265334 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278235 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204898 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207709 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02768 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018034 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177233 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180865 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016461 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201658 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297831 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362825 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0401655 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276421 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192614 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332774 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385769 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0359235 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328979 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321737 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215165 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210746 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187044 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157211 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171541 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295722 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227767 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178878 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0164505 s, 3.2 GB/s , 0.0168786 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310275 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155906 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176742 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236598 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182092 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180948 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020027 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376892 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038566 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414595 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404017 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297537 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306115 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231815 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0270182 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229523 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206838 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175617 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179151 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162487 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244118 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321858 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0213489 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419801 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374731 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392593 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0381607 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503307 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401137 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0536497 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244604 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429358 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526216 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435809 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302283 s, 1.7 GB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0369709 s, 1.4 GB/s , 0.0407385 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036376 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0271421 s, 1.9 GB/s , 0.0325989 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185012 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192033 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177004 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228066 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202183 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015928 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016541 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171155 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205294 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242831 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207968 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170273 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196374 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230365 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209162 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218109 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236764 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206873 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182859 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393397 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196782 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166218 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185375 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210196 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161883 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262839 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193727 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173858 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176189 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182467 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253903 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278869 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175081 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297409 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0332446 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226336 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.028008 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436442 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360106 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276259 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042826 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032616 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437342 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407912 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245665 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022993 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204786 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183552 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.027351 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0186267 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200719 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190681 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214956 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189681 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169521 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240194 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292078 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255425 s, 2.1 GB/s , 0.0399362 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317609 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03377 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280066 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488601 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399209 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041141 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311565 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318862 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270796 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377074 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296678 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423954 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385331 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355537 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396469 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278612 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324283 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0272242 s, 1.9 GB/s , 0.0208081 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257837 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038236 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330502 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658331 s, 796 MB/s 52428800 bytes (52 MB) copied, 0.0561851 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557891 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591194 s, 887 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868586 s, 604 MB/s , 0.103894 s, 505 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851743 s, 616 MB/s 52428800 bytes (52 MB) copied, 0.11709 s, 448 MB/s 52428800 bytes (52 MB) copied, 0.0921308 s, 569 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587265 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0983174 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12036 s, 436 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120541 s, 435 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729551 s, 719 MB/s 52428800 bytes (52 MB) copied, 0.118829 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565573 s, 927 MB/s 52428800 bytes (52 MB) copied, 0.0618733 s, 847 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918932 s, 571 MB/s 52428800 bytes (52 MB) copied, 0.0966121 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289109 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333931 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299808 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241779 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340218 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325192 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352916 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849998 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932669 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111051 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108915 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123459 s, 425 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128551 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116587 s, 450 MB/s 52428800 bytes (52 MB) copied, 0.106746 s, 491 MB/s 52428800 bytes (52 MB) copied, 0.124297 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127951 s, 410 MB/s , 0.136645 s, 384 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104477 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125833 s, 417 MB/s 52428800 bytes (52 MB) copied, 0.126864 s, 413 MB/s , 0.125464 s, 418 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132845 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103314 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105453 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342529 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458723 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237636 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357201 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235759 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336185 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258027 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353314 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398399 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050946 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515678 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750524 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702338 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731607 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750802 s, 698 MB/s 52428800 bytes (52 MB) copied, 0.115495 s, 454 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100292 s, 523 MB/s 52428800 bytes (52 MB) copied, 0.0781455 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138992 s, 377 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633571 s, 828 MB/s 52428800 bytes (52 MB) copied, 0.12192 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107033 s, 490 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0945312 s, 555 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663934 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064487 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0738247 s, 710 MB/s 52428800 bytes (52 MB) copied, 0.0865833 s, 606 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102583 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589016 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329175 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288256 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478766 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036636 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230031 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482241 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449971 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236256 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572696 s, 915 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291783 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0516916 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072262 s, 726 MB/s 52428800 bytes (52 MB) copied, 0.0737431 s, 711 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791615 s, 662 MB/s 52428800 bytes (52 MB) copied, 0.0726765 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0658367 s, 796 MB/s , 0.0364094 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342109 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100953 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109537 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0928752 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772199 s, 679 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729449 s, 719 MB/s 52428800 bytes (52 MB) copied, 0.0771703 s, 679 MB/s 52428800 bytes (52 MB) copied, 0.0506012 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080085 s, 655 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791069 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0668328 s, 784 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484487 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425547 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403717 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03911 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377039 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444644 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0467605 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294231 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391205 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341635 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579572 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780604 s, 672 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0935692 s, 560 MB/s 52428800 bytes (52 MB) copied, 0.0853851 s, 614 MB/s 52428800 bytes (52 MB) copied, 0.0851185 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994648 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893042 s, 587 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976175 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131025 s, 400 MB/s 52428800 bytes (52 MB) copied, 0.0911991 s, 575 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900131 s, 582 MB/s , 0.121732 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107492 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736123 s, 712 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0474366 s, 1.1 GB/s , 0.09729 s, 539 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918911 s, 571 MB/s 52428800 bytes (52 MB) copied, 0.14194 s, 369 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0813112 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334307 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759382 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439992 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042941 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260399 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349715 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233038 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248701 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267069 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448832 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362733 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572645 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600249 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617547 s, 849 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828314 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901044 s, 582 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167816 s, 312 MB/s 52428800 bytes (52 MB) copied, 0.170399 s, 308 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.109709 s, 478 MB/s , 0.151316 s, 346 MB/s , 0.124958 s, 420 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140481 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151565 s, 346 MB/s 52428800 bytes (52 MB) copied, 0.112465 s, 466 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135922 s, 386 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141061 s, 372 MB/s 52428800 bytes (52 MB) copied, 0.125133 s, 419 MB/s 52428800 bytes (52 MB) copied, 0.12858 s, 408 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121917 s, 430 MB/s 52428800 bytes (52 MB) copied, 0.1106 s, 474 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725257 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665894 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559869 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122494 s, 428 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116183 s, 451 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786599 s, 667 MB/s 52428800 bytes (52 MB) copied, 0.0846994 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873625 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679798 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593998 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400142 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343384 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301242 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386731 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292704 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320073 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630766 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262349 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416851 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350025 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556469 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291544 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239884 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322787 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357939 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780712 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0904657 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130227 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103369 s, 507 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115495 s, 454 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0993568 s, 528 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.135236 s, 388 MB/s 52428800 bytes (52 MB) copied, 0.115594 s, 454 MB/s 52428800 bytes (52 MB) copied, 0.119329 s, 439 MB/s 52428800 bytes (52 MB) copied, 0.133154 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159648 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147493 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13313 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139506 s, 376 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165639 s, 317 MB/s 52428800 bytes (52 MB) copied, 0.16682 s, 314 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122596 s, 428 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0823377 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133518 s, 393 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102478 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117048 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118238 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122963 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138714 s, 378 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128257 s, 409 MB/s 52428800 bytes (52 MB) copied, 0.135629 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103306 s, 508 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572417 s, 916 MB/s 52428800 bytes (52 MB) copied, 0.0563334 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0978945 s, 536 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059381 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502622 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328827 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357196 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238623 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230547 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028515 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288847 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240395 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284682 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019398 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209822 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374957 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041894 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339182 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412943 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232635 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391846 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318081 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349751 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338334 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208075 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362886 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281575 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339859 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247319 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0587961 s, 892 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288259 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0765633 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731054 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877896 s, 597 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051221 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806365 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886559 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108615 s, 483 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0887765 s, 591 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117484 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137423 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133601 s, 392 MB/s 52428800 bytes (52 MB) copied, 0.102198 s, 513 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151251 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092589 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11216 s, 467 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143416 s, 366 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11465 s, 457 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0912797 s, 574 MB/s 52428800 bytes (52 MB) copied, 0.109375 s, 479 MB/s 52428800 bytes (52 MB) copied, 0.11392 s, 460 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121512 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673364 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670502 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317017 s, 1.7 GB/s , 0.0676849 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937677 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671715 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0725274 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274566 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355558 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244161 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327761 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412926 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451516 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496545 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477954 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303378 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484353 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557349 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286923 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325898 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251768 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350294 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320726 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504766 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549801 s, 954 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571786 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0785391 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407315 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499666 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569846 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586422 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454563 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486451 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0462599 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588564 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651261 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627466 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617156 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548652 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652975 s, 803 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465927 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0751262 s, 698 MB/s , 0.0504183 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762714 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789779 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606424 s, 865 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114215 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789584 s, 664 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840125 s, 624 MB/s 52428800 bytes (52 MB) copied, 0.0763893 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108111 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107742 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084829 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119664 s, 438 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123017 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107805 s, 486 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694154 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734548 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729373 s, 719 MB/s 52428800 bytes (52 MB) copied, 0.068933 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0925596 s, 566 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975513 s, 537 MB/s 52428800 bytes (52 MB) copied, 0.0652906 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716481 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493871 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426221 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307283 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048473 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244072 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393512 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311842 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044525 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239735 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358616 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289668 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328135 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312257 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02017 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329889 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376189 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031723 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543418 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259716 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316288 s, 1.7 GB/s , 0.0282318 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225743 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206025 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025397 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501531 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468634 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412541 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617283 s, 849 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418416 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.052003 s, 1.0 GB/s , 0.0747909 s, 701 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504365 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0416746 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499312 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378656 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073657 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731387 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115783 s, 453 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0998387 s, 525 MB/s 52428800 bytes (52 MB) copied, 0.130469 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111643 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120432 s, 435 MB/s 52428800 bytes (52 MB) copied, 0.091567 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112991 s, 464 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931208 s, 563 MB/s , 0.130928 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0703349 s, 745 MB/s 52428800 bytes (52 MB) copied, 0.116016 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100344 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117712 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0936223 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353817 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115073 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563322 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369297 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250573 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329021 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258106 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381192 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262195 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464384 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235791 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394714 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203091 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346957 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211091 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300885 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026588 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311186 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229572 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224383 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249064 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030236 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325821 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.060919 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037722 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022889 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0424368 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021967 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368048 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231857 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248563 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269767 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399614 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447643 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0288013 s, 1.8 GB/s , 0.043172 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324248 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034687 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323936 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301058 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315409 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476573 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907533 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877336 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768671 s, 682 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0875897 s, 599 MB/s , 0.10785 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783494 s, 669 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125092 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128006 s, 410 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111052 s, 472 MB/s 52428800 bytes (52 MB) copied, 0.0904105 s, 580 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0975382 s, 538 MB/s 52428800 bytes (52 MB) copied, 0.0869573 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110869 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863069 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129229 s, 406 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054579 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078394 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794686 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413761 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568763 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417512 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644915 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0807907 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422845 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314039 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246868 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260579 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475583 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229118 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309965 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344492 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278372 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034035 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272585 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234069 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274273 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216059 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248275 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270732 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0206006 s, 2.5 GB/s , 0.0201375 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021812 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357092 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250294 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250478 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337955 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280876 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252556 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305235 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328988 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352225 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244858 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307084 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226184 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237302 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053674 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0622312 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065173 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.077834 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833725 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110811 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114848 s, 457 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937584 s, 559 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111558 s, 470 MB/s 52428800 bytes (52 MB) copied, 0.120294 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116604 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140655 s, 373 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143034 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11105 s, 472 MB/s 52428800 bytes (52 MB) copied, 0.122381 s, 428 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133649 s, 392 MB/s , 0.131807 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12375 s, 424 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543744 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149388 s, 351 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163711 s, 320 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542166 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12191 s, 430 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13031 s, 402 MB/s 52428800 bytes (52 MB) copied, 0.105905 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419676 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374645 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319683 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023755 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282781 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290814 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271147 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021142 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246369 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240998 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262414 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242312 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419249 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0313603 s, 1.7 GB/s , 0.0542753 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279512 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268078 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477494 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488551 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423501 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398445 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02819 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039907 s, 1.3 GB/s , 0.0358597 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242444 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238231 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02286 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211485 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217979 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212594 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221693 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234704 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253717 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02575 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298389 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330101 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248462 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200379 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0347762 s, 1.5 GB/s , 0.0221988 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416948 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476538 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0796626 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100576 s, 521 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101517 s, 516 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0863298 s, 607 MB/s , 0.101613 s, 516 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104214 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103033 s, 509 MB/s 52428800 bytes (52 MB) copied, 0.104624 s, 501 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101421 s, 517 MB/s 52428800 bytes (52 MB) copied, 0.10097 s, 519 MB/s 52428800 bytes (52 MB) copied, 0.106579 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102334 s, 512 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103912 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.089804 s, 584 MB/s , 0.0942214 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138102 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980321 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131025 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765889 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0884265 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572487 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361089 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325865 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395432 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303277 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395873 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026603 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244953 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289774 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221814 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629193 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318146 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566073 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261175 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291802 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241383 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362392 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416999 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263982 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339755 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694786 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397044 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693585 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318082 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476304 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356419 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024251 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338346 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678415 s, 773 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317124 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306335 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0532327 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280606 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275116 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337997 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025983 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271099 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028815 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475945 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039182 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349169 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03243 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0670515 s, 782 MB/s 52428800 bytes (52 MB) copied, 0.0855778 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366471 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509906 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662032 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640108 s, 819 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054433 s, 963 MB/s 52428800 bytes (52 MB) copied, 0.05332 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488939 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476248 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401419 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045604 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308807 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699165 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290215 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472342 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322234 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219642 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207296 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317795 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209096 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263193 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273546 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390812 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711536 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525771 s, 997 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0861032 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781021 s, 671 MB/s 52428800 bytes (52 MB) copied, 0.0853831 s, 614 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0621395 s, 844 MB/s 52428800 bytes (52 MB) copied, 0.0966666 s, 542 MB/s 52428800 bytes (52 MB) copied, 0.0915506 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730216 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865965 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734099 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798285 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078692 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460693 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0719742 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258664 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746531 s, 702 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612136 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308629 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600907 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299845 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510015 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233101 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435147 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384282 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485343 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070856 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572319 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924997 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079558 s, 659 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994802 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881613 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0927069 s, 566 MB/s 52428800 bytes (52 MB) copied, 0.115205 s, 455 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092348 s, 568 MB/s , 0.118201 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969786 s, 541 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113914 s, 460 MB/s , 0.0499119 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0784638 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730089 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561342 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424794 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263718 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263801 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269041 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288345 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247589 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254791 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218953 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019893 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231726 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275322 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0215729 s, 2.4 GB/s , 0.0392345 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285944 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253007 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305377 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402068 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390957 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0251208 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0315475 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495248 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231097 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306672 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042767 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335448 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495938 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448038 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044718 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0774738 s, 677 MB/s 52428800 bytes (52 MB) copied, 0.0598621 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671866 s, 780 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0923174 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11089 s, 473 MB/s , 0.0661776 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114597 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0867402 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100112 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608998 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0891614 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398729 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081723 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0912699 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0914991 s, 573 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366093 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341786 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0359679 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251257 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226361 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275961 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372935 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226075 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355263 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311467 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285185 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353318 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205846 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207446 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301099 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401535 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254862 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351526 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559084 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100494 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845238 s, 620 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0911844 s, 575 MB/s 52428800 bytes (52 MB) copied, 0.0831412 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840346 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966147 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727632 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932763 s, 562 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575996 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876256 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810727 s, 647 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.110629 s, 474 MB/s , 0.0883992 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0647973 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051727 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592413 s, 885 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405526 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411223 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286445 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260288 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316212 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243245 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247449 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289044 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241332 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300766 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435019 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269516 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284726 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318938 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033984 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431468 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445329 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256631 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340744 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023937 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212377 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316685 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473859 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661141 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450741 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699624 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868384 s, 604 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362046 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550535 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656748 s, 798 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609077 s, 861 MB/s 52428800 bytes (52 MB) copied, 0.0615639 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068573 s, 765 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121287 s, 432 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0784722 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655033 s, 800 MB/s 52428800 bytes (52 MB) copied, 0.116849 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546717 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106579 s, 492 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649801 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126578 s, 414 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112029 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758376 s, 691 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514306 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.049018 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407159 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268632 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236541 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272395 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374302 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022968 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262018 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432095 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296474 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293825 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457611 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798799 s, 656 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0669602 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538617 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845844 s, 620 MB/s 52428800 bytes (52 MB) copied, 0.0973873 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.07593 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101688 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111633 s, 470 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0623669 s, 841 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818163 s, 641 MB/s 52428800 bytes (52 MB) copied, 0.112669 s, 465 MB/s , 0.0806382 s, 650 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101573 s, 516 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062061 s, 845 MB/s 52428800 bytes (52 MB) copied, 0.107539 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065387 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132585 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662415 s, 791 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659429 s, 795 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0910488 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701932 s, 747 MB/s 52428800 bytes (52 MB) copied, 0.0889826 s, 589 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607966 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646614 s, 811 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619365 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624674 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291113 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252258 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247238 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231835 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032216 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023363 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279801 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308026 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331743 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376499 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439596 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436383 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430087 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415283 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0920285 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883695 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139419 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137414 s, 382 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120328 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128698 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0702875 s, 746 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161964 s, 324 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.148483 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159724 s, 328 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17538 s, 299 MB/s 52428800 bytes (52 MB) copied, 0.164763 s, 318 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718142 s, 730 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131868 s, 398 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0932869 s, 562 MB/s 52428800 bytes (52 MB) copied, 0.137032 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066028 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158597 s, 331 MB/s 52428800 bytes (52 MB) copied, 0.166033 s, 316 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.168416 s, 311 MB/s 52428800 bytes (52 MB) copied, 0.0757881 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432953 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038417 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558919 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560841 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360429 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288773 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491437 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444731 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359563 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244314 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237836 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251598 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208237 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033107 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257908 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447475 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324722 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362483 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401304 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389667 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293449 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344785 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298946 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271369 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0252248 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021965 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381908 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493822 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0679783 s, 771 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762233 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.090628 s, 579 MB/s 52428800 bytes (52 MB) copied, 0.106829 s, 491 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111989 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130261 s, 402 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140398 s, 373 MB/s 52428800 bytes (52 MB) copied, 0.146462 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193659 s, 271 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188983 s, 277 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137229 s, 382 MB/s 52428800 bytes (52 MB) copied, 0.15294 s, 343 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.12735 s, 412 MB/s , 0.14835 s, 353 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137592 s, 381 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153743 s, 341 MB/s 52428800 bytes (52 MB) copied, 0.156812 s, 334 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124664 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161537 s, 325 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967483 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109288 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.169346 s, 310 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.195064 s, 269 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165844 s, 316 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705221 s, 743 MB/s 52428800 bytes (52 MB) copied, 0.0753181 s, 696 MB/s 52428800 bytes (52 MB) copied, 0.176612 s, 297 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103278 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682341 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811481 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0729446 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135599 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406731 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338424 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038428 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503125 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336792 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227503 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035589 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038578 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493463 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778201 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382075 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560934 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579979 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580204 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888146 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328996 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064794 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304574 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475542 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362965 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756967 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0907769 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415726 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585236 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466492 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445429 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0874378 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247585 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245827 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630752 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459213 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283159 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393124 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340177 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320338 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280939 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315481 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264662 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461159 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423795 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351525 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392722 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435085 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417748 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220516 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230358 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272068 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270949 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025392 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229578 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253936 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030852 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258092 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380156 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427108 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336743 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448951 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364363 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0217464 s, 2.4 GB/s , 0.0284923 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278112 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265537 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262863 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289145 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182465 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231302 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257097 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400625 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309265 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534056 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464868 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728956 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614551 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504678 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0646094 s, 811 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0886093 s, 592 MB/s , 0.0949264 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642456 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485753 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660269 s, 794 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833271 s, 629 MB/s 52428800 bytes (52 MB) copied, 0.0687284 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0996004 s, 526 MB/s 52428800 bytes (52 MB) copied, 0.0843684 s, 621 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768954 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0801369 s, 654 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11759 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0915799 s, 572 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131984 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116259 s, 451 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115051 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116949 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114316 s, 459 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133636 s, 392 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104114 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106061 s, 494 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101885 s, 515 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819798 s, 640 MB/s 52428800 bytes (52 MB) copied, 0.0833684 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847129 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0836677 s, 627 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603611 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072925 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444251 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311841 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474807 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250459 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279457 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276829 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030318 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300131 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243472 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287167 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315036 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531642 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597836 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055301 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644921 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953158 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684363 s, 766 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102775 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.087742 s, 598 MB/s 52428800 bytes (52 MB) copied, 0.0810532 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682324 s, 768 MB/s 52428800 bytes (52 MB) copied, 0.11167 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131985 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0819423 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121906 s, 430 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118942 s, 441 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840291 s, 624 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.104426 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092669 s, 566 MB/s , 0.0629694 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464896 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494894 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0929358 s, 564 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486069 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308374 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309507 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330268 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391548 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268955 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0457007 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309233 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034761 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0480852 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255339 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272167 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288609 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0321642 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334813 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284381 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340383 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279838 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336549 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312424 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388424 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407481 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267728 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491316 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530279 s, 989 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779696 s, 672 MB/s 52428800 bytes (52 MB) copied, 0.086996 s, 603 MB/s 52428800 bytes (52 MB) copied, 0.0931991 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765309 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742235 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931935 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103985 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100878 s, 520 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081215 s, 646 MB/s 52428800 bytes (52 MB) copied, 0.104058 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0926425 s, 566 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104464 s, 502 MB/s 52428800 bytes (52 MB) copied, 0.0948097 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637935 s, 822 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0934244 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888591 s, 590 MB/s 52428800 bytes (52 MB) copied, 0.110789 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078652 s, 667 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578401 s, 906 MB/s 52428800 bytes (52 MB) copied, 0.100385 s, 522 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434551 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404215 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0909976 s, 576 MB/s 52428800 bytes (52 MB) copied, 0.104407 s, 502 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569311 s, 921 MB/s 52428800 bytes (52 MB) copied, 0.0468615 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596532 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418075 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368311 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305882 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288703 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366651 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346414 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450542 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346553 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516413 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304071 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052138 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0802863 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536918 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663374 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599895 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465253 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547093 s, 958 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307795 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367075 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438948 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0359336 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323098 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033868 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374868 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0506464 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906306 s, 578 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651434 s, 805 MB/s 52428800 bytes (52 MB) copied, 0.0926444 s, 566 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0840662 s, 624 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875985 s, 599 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952236 s, 551 MB/s 52428800 bytes (52 MB) copied, 0.1394 s, 376 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118484 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112494 s, 466 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101481 s, 517 MB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0953014 s, 550 MB/s , 0.102518 s, 511 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129543 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882123 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776616 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724427 s, 724 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130154 s, 403 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399936 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182716 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109157 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566021 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125227 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615724 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457421 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033898 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253933 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503697 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0616954 s, 850 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477107 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0598028 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651464 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503601 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651526 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0866946 s, 605 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623328 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732125 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0765363 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832422 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078617 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710538 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583115 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557559 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0936022 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776468 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779123 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492373 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477399 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318277 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217319 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458915 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0531767 s, 986 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535603 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0992827 s, 528 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107316 s, 489 MB/s 52428800 bytes (52 MB) copied, 0.0868636 s, 604 MB/s 52428800 bytes (52 MB) copied, 0.116624 s, 450 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0997064 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104064 s, 504 MB/s 52428800 bytes (52 MB) copied, 0.100908 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124596 s, 421 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106927 s, 490 MB/s 52428800 bytes (52 MB) copied, 0.0959924 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136594 s, 384 MB/s 52428800 bytes (52 MB) copied, 0.127566 s, 411 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135286 s, 388 MB/s 52428800 bytes (52 MB) copied, 0.105926 s, 495 MB/s 52428800 bytes (52 MB) copied, 0.1303 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112507 s, 466 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901966 s, 581 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0976691 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779914 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464246 s, 1.1 GB/s , 0.0627823 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631483 s, 830 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052952 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247882 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278789 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028232 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207754 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237631 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225362 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021738 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258003 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293681 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247948 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250772 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336865 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266647 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327149 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031526 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300021 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029083 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333001 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654339 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656524 s, 799 MB/s 52428800 bytes (52 MB) copied, 0.075227 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816581 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0786699 s, 666 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775444 s, 676 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105465 s, 497 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0760819 s, 689 MB/s 52428800 bytes (52 MB) copied, 0.10577 s, 496 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0987368 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121872 s, 430 MB/s 52428800 bytes (52 MB) copied, 0.113391 s, 462 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114087 s, 460 MB/s 52428800 bytes (52 MB) copied, 0.113643 s, 461 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109308 s, 480 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11434 s, 459 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.142883 s, 367 MB/s 52428800 bytes (52 MB) copied, 0.107184 s, 489 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105607 s, 496 MB/s 52428800 bytes (52 MB) copied, 0.0778759 s, 673 MB/s 52428800 bytes (52 MB) copied, 0.128483 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118496 s, 442 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116308 s, 451 MB/s 52428800 bytes (52 MB) copied, 0.141269 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104334 s, 503 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686945 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811941 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0893297 s, 587 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103065 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470518 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058453 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474332 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0968316 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461048 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557767 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314913 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057734 s, 908 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345007 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371121 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234363 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0259748 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0278407 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335574 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532286 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285789 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307555 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048073 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0595328 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470832 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468808 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568617 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035349 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659822 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655923 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746174 s, 703 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649021 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250519 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563558 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811239 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100895 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453213 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573094 s, 915 MB/s /mnt/lustre/f45.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x1a5:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff88012b636000.stats= snapshot_time 1746141526.752202973 secs.nsecs start_time 1746141466.204174493 secs.nsecs elapsed_time 60.548028480 secs.nsecs read_bytes 1050 samples [bytes] 1048576 1048576 1101004800 1154487209164800 read 143150 samples [usecs] 138 214975 150736124 2175556486150 open 2863 samples [usecs] 3 7346 81796 196550254 close 2863 samples [usecs] 7 26132 400852 3700866206 seek 2862 samples [usecs] 2 8063 36427 111532327 inode_permission 5724 samples [usecs] 1 18260 453634 1130549572 opencount 2863 samples [reqs] 1 2847 4054144 7696086336 openclosetime 2846 samples [usecs] 40 143309 55406304 2087722185502 pcc_attach 1 samples [reqs] pcc_hit_bytes 2842 samples [bytes] 52428800 52428800 149002649600 7812030115348480000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff88012e148000.stats= snapshot_time 1746141526.752311358 secs.nsecs start_time 1746141466.204197653 secs.nsecs elapsed_time 60.548113705 secs.nsecs read_bytes 1053 samples [bytes] 90112 1048576 1101004800 1153404541861888 read 145750 samples [usecs] 140 252407 149765763 1761603269131 ioctl 1 samples [reqs] open 2916 samples [usecs] 5 68580 812794 35074418084 close 2916 samples [usecs] 8 28056 374694 2679832562 seek 2914 samples [usecs] 2 9053 38012 165719574 setattr 1 samples [usecs] 16600 16600 16600 275560000 getattr 32 samples [usecs] 80 2563 10740 12362480 mknod 1 samples [usecs] 9427 9427 9427 88868329 inode_permission 11641 samples [usecs] 1 15041 1114422 1104528734 opencount 2917 samples [reqs] 1 2916 4252987 8269222447 openclosetime 2915 samples [usecs] 18 179361 56179865 2341192948353 pcc_attach 1 samples [reqs] pcc_hit_bytes 2894 samples [bytes] 52428800 52428800 151728947200 7954966626959360000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 attach 2 detach 0 auto_attach 0 llite.lustre-ffff88012b636000.pcc_async_threshold=1073741824 llite.lustre-ffff88012e148000.pcc_async_threshold=1073741824 /mnt/pcc.d45.sanity-pcc.2 is idle now /mnt/pcc.d45.sanity-pcc is idle now PASS 45 (65s) == sanity-pcc test 46: Verify PCC mode setting works correctly ========================================================== 19:18:49 (1746141529) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0841528 s, 748 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d46.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 5ba2ae87-a55a-46d4-88f7-8da358aceb6c Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f46.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5ba2ae87-a55a-46d4-88f7-8da358aceb6c (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d46.sanity-pcc/d46.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} running as uid/gid/euid/egid 500/500/500/500, groups: 0 [id] uid=500(sanityusr) gid=500(sanityusr) groups=500(sanityusr),0(root) Mode: 0400 llite.lustre-ffff88012e148000.pcc_mode=0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff88012e148000.pcc_mode=0400 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff88012e148000.pcc_mode=0004 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [stat] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 1 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0664/-rw-rw-r--) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-01 19:18:50.000000000 -0400 Modify: 2025-05-01 19:18:50.000000000 -0400 Change: 2025-05-01 19:18:50.000000000 -0400 Birth: - running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff88012e148000.pcc_mode=0040 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 4 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0660/-rw-rw----) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-01 19:18:50.000000000 -0400 Modify: 2025-05-01 19:18:50.000000000 -0400 Change: 2025-05-01 19:18:50.000000000 -0400 Birth: - file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff88012e148000.pcc_mode=0400 llite.lustre-ffff88012e148000.pcc_mode=0400 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d46.sanity-pcc is idle now PASS 46 (3s) == sanity-pcc test 47: mtime should be kept once file attached into PCC ========================================================== 19:18:52 (1746141532) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0794903 s, 791 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d47.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 3e38dfc1-cd9c-4bb9-9d2d-09bcbfc99cb4 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f47.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3e38dfc1-cd9c-4bb9-9d2d-09bcbfc99cb4 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d47.sanity-pcc/d47.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} QQQQQ /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/f47.sanity-pcc | grep -E -c 'type: readonly' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d47.sanity-pcc is idle now PASS 47 (6s) == sanity-pcc test 48: PCC state should check whether the file in local PCC cache ========================================================== 19:18:58 (1746141538) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0836192 s, 752 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d48.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 1039a837-6238-42ab-a34c-46a2f64d07d7 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f48.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1039a837-6238-42ab-a34c-46a2f64d07d7 (extents) (64bit) (large files) (huge files) QQQQQ file: /mnt/lustre/f48.sanity-pcc, type: readonly, PCC_file: /d48.sanity-pcc/59fd/0x2000059f1:0xc:0x0 (unlinked), open_count: 0, flags: 4 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d48.sanity-pcc is idle now PASS 48 (4s) == sanity-pcc test 96: Auto attach from multiple read process on a node ========================================================== 19:19:02 (1746141542) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0941921 s, 668 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d96.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: af9972c3-f48f-4db4-9b4c-c08b91d969b5 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f96.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=af9972c3-f48f-4db4-9b4c-c08b91d969b5 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d96.sanity-pcc/d96.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff88012b636000.pcc_async_threshold=1G llite.lustre-ffff88012e148000.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.429587 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.586928 s, 89.3 MB/s , 0.610195 s, 85.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.651699 s, 80.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.612673 s, 85.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.614733 s, 85.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462787 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322338 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375921 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303959 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02954 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234733 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260039 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432846 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247335 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025825 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401254 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355683 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271446 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433178 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396735 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336498 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045285 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393379 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300308 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284902 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355507 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297597 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203906 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215318 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405846 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403968 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349232 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326025 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291662 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222632 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383132 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255642 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251133 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246427 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254703 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240431 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310778 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314871 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390947 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488389 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329926 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295042 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291468 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311054 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223257 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051397 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338907 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319037 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216947 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297835 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271798 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324304 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344737 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241464 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258106 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386983 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034274 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266067 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241229 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363176 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395211 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318442 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272759 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260773 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220423 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300619 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356167 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390185 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287902 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290571 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036691 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368382 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309097 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414769 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409545 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364624 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365197 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235971 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347303 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289766 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321655 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517283 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291124 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314893 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224965 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268716 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317301 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187939 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252401 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277984 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283226 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215602 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275281 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233751 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351082 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285444 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022038 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228901 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230101 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219517 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248234 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299065 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425435 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288469 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247958 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290501 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420914 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271841 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320583 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039307 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412297 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268484 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339434 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221006 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200159 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314591 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536508 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269817 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364709 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332124 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203547 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237318 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206537 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021353 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307971 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232393 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239929 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210599 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344786 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351039 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316907 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334083 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303039 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295172 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297142 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264147 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227124 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283558 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273662 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252452 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353245 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361728 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031019 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262715 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349623 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330296 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299683 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237738 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274526 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207379 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310309 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449691 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301283 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322793 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386454 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339935 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228583 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306906 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255243 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264066 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337271 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255338 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328045 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303934 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242369 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214803 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027882 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197857 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031288 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265184 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306774 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482707 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407776 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027787 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305242 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438999 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420269 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197311 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368353 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333171 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216534 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323015 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345775 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380644 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268087 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247795 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319388 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366996 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223282 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021604 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234603 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273752 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026793 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237641 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255326 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226119 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212807 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280522 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280212 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345694 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299304 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212653 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224866 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214063 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019648 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299736 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295133 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374387 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222475 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190909 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267617 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291145 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209358 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250604 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225544 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225701 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206142 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019679 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270975 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207712 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030523 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300704 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240061 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278921 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212848 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318989 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256305 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341096 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318737 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306201 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342056 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262623 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250657 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316415 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275051 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219218 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240682 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249248 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325976 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241305 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197918 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287469 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036514 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166393 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235639 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204194 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269138 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229681 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247482 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255546 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225709 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225682 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254408 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252721 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316527 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287945 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289108 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325841 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293238 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169586 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175386 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319224 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194616 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244419 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280648 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223627 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227161 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174096 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296104 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315804 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179433 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329322 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247815 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341619 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022241 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233034 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182917 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0145173 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163496 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213095 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196821 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269048 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211894 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227538 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353481 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388958 s, 1.3 GB/s , 0.0394519 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337872 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277607 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227982 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257982 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183251 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185388 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371787 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184788 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211709 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353579 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297873 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309047 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179205 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237903 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261929 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179233 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195443 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214067 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02188 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265125 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02273 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241804 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303805 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243776 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215643 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269063 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227862 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253505 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222903 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319185 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284714 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281184 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437108 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026008 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345836 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244824 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173378 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262316 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164364 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287727 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361634 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231682 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242107 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021032 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180582 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258153 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276392 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312805 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267963 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272248 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248286 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361948 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286516 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322385 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341132 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330895 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298286 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565705 s, 927 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019902 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287397 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381756 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189671 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216112 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326909 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311409 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376845 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344305 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333612 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455339 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200788 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430209 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456352 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331199 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413667 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393477 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023556 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277779 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232341 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533167 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349812 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576871 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371979 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313266 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422399 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324245 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230513 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362527 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028332 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321629 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454991 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371742 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370937 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424401 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021679 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465948 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302702 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218835 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204283 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395382 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262792 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238425 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214322 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233545 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259201 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344129 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254815 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278679 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311553 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343202 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266645 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332745 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229824 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346524 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197793 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241437 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208125 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247009 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260976 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298734 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306702 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320653 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481654 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263205 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225627 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250546 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318435 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323514 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295256 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277983 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216356 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252823 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273516 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307693 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228351 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027181 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221396 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229772 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256628 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031529 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311035 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046451 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226794 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186944 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289084 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022161 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221581 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252555 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02165 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235448 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376879 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036538 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283987 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290611 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212993 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220631 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238898 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274916 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259742 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02664 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322418 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317343 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311472 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268661 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245916 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246266 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209247 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184079 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325476 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173245 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025666 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228651 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234972 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308779 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235275 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195377 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209347 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181704 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270731 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203042 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02348 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499471 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328623 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579905 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291021 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370797 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254969 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026623 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340564 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244258 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196733 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030477 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285441 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033309 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289057 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194861 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261991 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290738 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029319 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350461 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218065 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194536 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307014 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217307 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189111 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256027 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215455 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029011 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188842 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222272 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331898 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338577 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252005 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022515 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204861 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257608 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349197 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513911 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307165 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324781 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284913 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207833 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277105 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408179 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186419 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239213 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175599 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184067 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320811 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277241 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207219 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303362 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463679 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217228 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388641 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508945 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447917 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366224 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02164 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223016 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251109 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289625 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287447 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248552 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191434 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279179 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025714 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402774 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230806 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265281 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021969 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372061 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212371 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258711 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271888 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232071 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278974 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207214 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305536 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204581 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237068 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020251 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406691 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209187 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288749 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332418 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245462 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197946 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260232 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239713 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201848 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237194 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189282 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221976 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270611 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299239 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244476 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247372 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185388 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272179 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208764 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206997 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025726 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021894 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225549 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020286 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367615 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315474 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362815 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237687 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190782 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024301 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402496 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203813 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276067 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255119 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267165 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243668 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276374 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235531 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208068 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.031674 s, 1.7 GB/s , 0.0273543 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344714 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225362 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278095 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357482 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255223 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327261 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177806 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293405 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283408 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234773 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239008 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371099 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227116 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400005 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179708 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316726 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172096 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276588 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222239 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188216 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312385 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250265 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223837 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273517 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227833 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403101 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409089 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282858 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301259 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270884 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245319 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211156 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031154 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035142 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220897 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241537 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026745 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246483 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287844 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198183 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0350527 s, 1.5 GB/s , 0.0308238 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032489 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191904 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381981 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267846 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323017 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217747 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177039 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217447 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178716 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242169 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021306 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294496 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262288 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235174 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285165 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258424 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317827 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225459 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235661 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231888 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204111 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178846 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214042 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016508 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205685 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200204 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227775 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218975 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302433 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221809 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176943 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230274 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346452 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320748 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673324 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397352 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265787 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384491 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258468 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211645 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242789 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227257 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202318 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242975 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319573 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299341 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187443 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019201 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03199 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302641 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286484 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289145 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236674 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217801 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174519 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247056 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229176 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209418 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376673 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251162 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205423 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279931 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180793 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352017 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019807 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204869 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212848 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027515 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412733 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514975 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477861 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526049 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361628 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308708 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185699 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328645 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488124 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023178 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285407 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212684 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225839 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272355 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432136 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358563 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230742 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236451 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191855 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417457 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338234 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233059 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386879 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191539 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028878 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330865 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339362 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259523 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218772 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320001 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367484 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350625 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232654 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322144 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217501 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305533 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339336 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267647 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175653 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267026 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214579 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190415 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245391 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342705 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354484 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303019 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046455 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254286 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0312882 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0544797 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371002 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183192 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600937 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401052 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220373 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231137 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254704 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248483 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182496 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220088 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281735 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238928 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166216 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373049 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217945 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285059 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335653 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260575 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322016 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023213 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024962 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351797 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215568 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021604 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026836 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281014 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174833 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280757 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226076 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230689 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249046 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228523 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249062 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191006 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275825 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267014 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204942 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191263 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160712 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151179 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180599 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195771 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213499 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239008 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480341 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270299 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305301 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362106 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207926 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318424 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027169 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240086 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025897 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376572 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291872 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227913 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369467 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341993 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316029 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328434 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264052 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209129 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225749 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360748 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031173 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040062 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339331 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254114 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026598 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263981 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029627 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233842 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373049 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260847 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203147 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272982 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301767 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350615 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206052 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216312 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264207 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293781 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249685 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228398 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260112 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257684 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220331 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034382 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248975 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038862 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219624 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0237365 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227333 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272531 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216587 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242681 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298881 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408306 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251343 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364944 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412198 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438204 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276882 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383401 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251827 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237849 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209376 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239397 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384262 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270586 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411972 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279776 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214121 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285541 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282431 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343829 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193349 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506433 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219478 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649997 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031298 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269469 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429923 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224253 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493371 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442528 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255182 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262928 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271317 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319889 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277705 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238288 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427082 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032058 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317101 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181058 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026518 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220459 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356982 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386244 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493825 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451917 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327617 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278567 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415847 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040989 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405651 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246921 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437994 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316026 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336035 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182681 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343084 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324255 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280219 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344194 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300896 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0543656 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319589 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107407 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298263 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0665383 s, 788 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238788 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301207 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366331 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354982 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242959 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371525 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347467 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430565 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277122 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200915 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288414 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252453 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313856 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438852 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291486 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398148 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278642 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244574 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272084 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223847 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209949 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025901 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280042 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231958 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180103 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301211 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024502 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182184 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462156 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339059 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282613 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398163 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296086 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253391 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221206 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282561 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234451 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226757 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284404 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232941 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486752 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342649 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307337 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233489 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02521 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258364 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192785 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020459 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329715 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382123 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288853 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255263 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384886 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242574 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324392 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382261 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291521 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313731 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445295 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055493 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457634 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293859 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361054 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419564 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279677 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239805 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035637 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714249 s, 734 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574435 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331207 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276012 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282481 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494082 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362524 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324783 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426212 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376866 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260952 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033302 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033605 s, 1.6 GB/s , 0.0223871 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194799 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277863 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023221 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042132 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427771 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236893 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324047 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278435 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232787 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255147 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218418 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455445 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285184 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204702 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022548 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224907 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335477 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251639 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02492 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230208 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199325 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028717 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356366 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213028 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240023 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358672 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229619 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297271 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293535 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238059 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386348 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291575 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246749 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026351 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265518 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294771 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235786 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270979 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355947 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432405 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585123 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202169 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395755 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270996 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019583 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046875 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616155 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253049 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313837 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377846 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294306 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289248 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396986 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228855 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254652 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288951 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328953 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327423 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301539 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268402 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344785 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425413 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273603 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323863 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029897 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026132 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252619 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359131 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466882 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449924 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230166 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303983 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373045 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02823 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429464 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205449 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190969 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196079 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281052 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224592 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340338 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218913 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299654 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234805 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238837 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255759 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305279 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243913 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289572 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224407 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030345 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284645 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346658 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336698 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258749 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224417 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215188 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390535 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404643 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480711 s, 1.1 GB/s Finish ======== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367476 s, 1.4 GB/s detach: must specify one or more FIDs Detach given files from the Persistent Client Cache. usage: lfs pcc detach [-k] FILE [...] lfs commands used to interact with PCC features: lfs pcc attach - attach given files to Persistent Client Cache lfs pcc attach_fid - attach given files into PCC by FID(s) lfs pcc state - display the PCC state for given files lfs pcc detach - detach given files from Persistent Client Cache lfs pcc detach_fid - detach given files from PCC by FID(s) lfs pcc delete - delete the PCC layout componenet for given files lfs pcc pin - pin give files for PCC lfs pcc unpin - unpin given files for PCC llite.lustre-ffff88012b636000.pcc_async_threshold=1073741824 llite.lustre-ffff88012e148000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d96.sanity-pcc is idle now PASS 96 (64s) == sanity-pcc test 97: two mmap I/O and layout lock cancel ========================================================== 19:20:06 (1746141606) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0774134 s, 813 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d97.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: f8068092-8b74-499e-ad01-1738c584f4f5 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f97.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f8068092-8b74-499e-ad01-1738c584f4f5 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d97.sanity-pcc/d97.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff88012b636000.pcc_async_threshold=1G llite.lustre-ffff88012e148000.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.319865 s, 164 MB/s T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... llite.lustre-ffff88012b636000.pcc_async_threshold=1073741824 llite.lustre-ffff88012e148000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d97.sanity-pcc is idle now PASS 97 (126s) == sanity-pcc test 98: racer between auto attach and mmap I/O ========================================================== 19:22:12 (1746141732) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0996744 s, 631 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d98.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 153221df-f8d1-4f7f-bc27-bdfc4d116cf6 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f98.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=153221df-f8d1-4f7f-bc27-bdfc4d116cf6 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d98.sanity-pcc/d98.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff88012b636000.pcc_async_threshold=0 llite.lustre-ffff88012e148000.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.497531 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.524839 s, 99.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.613715 s, 85.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.568689 s, 92.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.18744 s, 44.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.19179 s, 44.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.18879 s, 44.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499255 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630451 s, 832 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384425 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0794256 s, 660 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.621601 s, 84.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457623 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.263549 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.749671 s, 69.9 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.05894 s, 49.5 MB/s 52428800 bytes (52 MB) copied, 0.858517 s, 61.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.414318 s, 127 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.289254 s, 181 MB/s 52428800 bytes (52 MB) copied, 0.442685 s, 118 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.254368 s, 206 MB/s , 0.340341 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.242184 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218227 s, 240 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.381862 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.384882 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387557 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239687 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.451707 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.532045 s, 98.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.488719 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.565886 s, 92.6 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.158101 s, 332 MB/s , 0.140709 s, 373 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0873921 s, 600 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.881948 s, 59.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.891954 s, 58.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.09719 s, 47.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.23098 s, 227 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162604 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.249547 s, 210 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.474148 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.518296 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267878 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.223144 s, 235 MB/s , 0.158692 s, 330 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.375109 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.380086 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.563607 s, 93.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.485499 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351927 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.571481 s, 91.7 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.262989 s, 199 MB/s , 0.404312 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193976 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677261 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104093 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352976 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.501189 s, 105 MB/s 52428800 bytes (52 MB) copied, 0.484027 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.57452 s, 91.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188723 s, 278 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.193342 s, 271 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.350301 s, 150 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.025014 s, 2.1 GB/s , 0.0356108 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080439 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.537489 s, 97.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.416325 s, 126 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443484 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.372023 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.372103 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.28596 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182778 s, 287 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278159 s, 188 MB/s , 0.0654583 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.586705 s, 89.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.484433 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.485589 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158787 s, 330 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069651 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754527 s, 695 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.386792 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366157 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105065 s, 499 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061551 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.504862 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.638428 s, 82.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.425672 s, 123 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.675826 s, 77.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.678625 s, 77.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.470475 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206201 s, 254 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.382915 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.53763 s, 97.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124705 s, 420 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.246131 s, 213 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.386289 s, 136 MB/s , 0.370103 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.406812 s, 129 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.157854 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.459713 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.361866 s, 145 MB/s , 0.42854 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.430242 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131414 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059189 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385777 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.66035 s, 79.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.603018 s, 86.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.50239 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.426621 s, 123 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243513 s, 215 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.380777 s, 138 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.189913 s, 276 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.736714 s, 71.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.666537 s, 78.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847258 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453954 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045864 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258208 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0259185 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511207 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298327 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285407 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461222 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393376 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282657 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413523 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358976 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550511 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386012 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451831 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439947 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378874 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377403 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530418 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0559061 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267551 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301049 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459969 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.305246 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160082 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.420163 s, 125 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.471914 s, 111 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.480792 s, 109 MB/s 52428800 bytes (52 MB) copied, 0.336881 s, 156 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.354286 s, 148 MB/s , 0.219471 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392126 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0913609 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966213 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359158 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577922 s, 90.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.582775 s, 90.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.513391 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.345718 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.586873 s, 89.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.34986 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.404954 s, 129 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207454 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22908 s, 229 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.432959 s, 121 MB/s , 0.117148 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.437264 s, 120 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.788782 s, 66.5 MB/s 52428800 bytes (52 MB) copied, 0.797253 s, 65.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.801444 s, 65.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.482512 s, 109 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.483296 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.501281 s, 105 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186522 s, 281 MB/s 52428800 bytes (52 MB) copied, 0.327206 s, 160 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533467 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513316 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651649 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.12307 s, 46.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431658 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604616 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415671 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317876 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374379 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399153 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398914 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278737 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307916 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508469 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479938 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492967 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062195 s, 843 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676259 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522272 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355383 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338968 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424585 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314804 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051793 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297285 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511986 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297447 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329133 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318647 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.74517 s, 70.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.346224 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.509407 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0998337 s, 525 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141175 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.428254 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134925 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.73391 s, 71.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.78512 s, 66.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.606506 s, 86.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318688 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.435974 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.453098 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.187259 s, 280 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.255063 s, 206 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.770787 s, 68.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.987442 s, 53.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.584409 s, 89.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.926334 s, 56.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130334 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247955 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.849167 s, 61.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.856937 s, 61.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.818503 s, 64.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264685 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.3245 s, 162 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326387 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.450295 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.358778 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.452272 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387773 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401393 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.528642 s, 99.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.526572 s, 99.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.961683 s, 54.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280458 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.590507 s, 88.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409859 s, 128 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.437112 s, 120 MB/s , 0.108111 s, 485 MB/s , 0.0471372 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144732 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.149089 s, 352 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.657606 s, 79.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342753 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.438552 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184174 s, 285 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.505047 s, 104 MB/s , 0.502747 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.848705 s, 61.8 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.04447 s, 50.2 MB/s 52428800 bytes (52 MB) copied, 1.04893 s, 50.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184364 s, 284 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.775078 s, 67.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.07099 s, 49.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.990534 s, 52.9 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.43615 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.556688 s, 94.2 MB/s 52428800 bytes (52 MB) copied, 0.655936 s, 79.9 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.120285 s, 436 MB/s , 0.120376 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.264921 s, 198 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.581113 s, 90.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.13059 s, 46.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.04021 s, 50.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.870919 s, 60.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.920044 s, 57.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.879872 s, 59.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.127833 s, 410 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.333571 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177889 s, 295 MB/s llite.lustre-ffff88012b636000.pcc_async_threshold=1073741824 llite.lustre-ffff88012e148000.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d98.sanity-pcc is idle now PASS 98 (66s) == sanity-pcc test 99: race among unlink | mmap read | write | detach for PCC-RO file ========================================================== 19:23:18 (1746141798) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.103668 s, 607 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 9e1b046e-3d40-440f-8462-6949eeb4dbcf Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f99.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=9e1b046e-3d40-440f-8462-6949eeb4dbcf (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.551878 s, 95.0 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.772758 s, 67.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367358 s, 1.4 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.597144 s, 87.8 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.918153 s, 57.1 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.05582 s, 49.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02344 s, 51.2 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.2232 s, 42.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.347724 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.604707 s, 86.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.38819 s, 37.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672609 s, 779 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.921957 s, 56.9 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667202 s, 786 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.784405 s, 66.8 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116195 s, 451 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.809649 s, 64.8 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.842292 s, 62.2 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.962329 s, 54.5 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.754155 s, 69.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363445 s, 1.4 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.610606 s, 85.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.875911 s, 59.9 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.1419 s, 45.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354189 s, 1.5 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.16369 s, 45.1 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541089 s, 969 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.881013 s, 59.5 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02688 s, 51.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.2226 s, 42.9 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.897862 s, 58.4 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282405 s, 1.9 GB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.576599 s, 90.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.858749 s, 61.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.970133 s, 54.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.674641 s, 77.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.51941 s, 34.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.593855 s, 88.3 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.732967 s, 71.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.620367 s, 84.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.696082 s, 75.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.983083 s, 53.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408017 s, 1.3 GB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456709 s, 115 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.822821 s, 63.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.903238 s, 58.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.558049 s, 94.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336931 s, 1.6 GB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.41029 s, 128 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.859737 s, 61.0 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294052 s, 1.8 GB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.531279 s, 98.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.329071 s, 159 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.02525 s, 51.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238085 s, 2.2 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.597558 s, 87.7 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.227594 s, 230 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.92549 s, 56.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.464963 s, 113 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.866643 s, 60.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.919524 s, 57.0 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.699361 s, 75.0 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.962449 s, 54.5 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.745498 s, 70.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223295 s, 2.3 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.788112 s, 66.5 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.863359 s, 60.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.864621 s, 60.6 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550382 s, 953 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.528085 s, 99.3 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.708303 s, 74.0 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.548956 s, 95.5 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.710853 s, 73.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.550549 s, 95.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186955 s, 2.8 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.412205 s, 127 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.448311 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.46059 s, 35.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515405 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.752282 s, 69.7 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.525426 s, 99.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448891 s, 1.2 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... ==== DONE ==== unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.486285 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.763729 s, 68.6 MB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff88012b636000.rpc_stats= snapshot_time: 1746141862.231187940 secs.nsecs start_time: 1746141330.275493023 secs.nsecs elapsed_time: 531.955694917 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 2 4 4 | 0 0 0 2: 1 2 6 | 0 0 0 3: 1 2 8 | 0 0 0 4: 1 2 10 | 0 0 0 5: 1 2 12 | 0 0 0 6: 1 2 14 | 0 0 0 7: 1 2 16 | 0 0 0 8: 2 4 20 | 0 0 0 9: 40 80 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0000-osc-ffff88012e148000.rpc_stats= snapshot_time: 1746141862.231343578 secs.nsecs start_time: 1746141328.510253123 secs.nsecs elapsed_time: 533.721090455 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 0 0 0 | 0 0 1 32: 0 0 0 | 0 0 1 64: 0 0 0 | 0 0 1 128: 0 0 0 | 0 0 1 256: 352 99 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 36 10 10 | 9 5 5 2: 27 7 17 | 37 24 30 3: 18 5 22 | 47 30 60 4: 18 5 27 | 27 17 78 5: 13 3 31 | 11 7 85 6: 13 3 35 | 11 7 92 7: 13 3 38 | 10 6 99 8: 22 6 45 | 1 0 100 9: 195 54 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 28 7 17 | 12 7 17 2048: 56 15 32 | 24 15 33 4096: 112 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff88012b636000.rpc_stats= snapshot_time: 1746141862.231403186 secs.nsecs start_time: 1746141330.276184014 secs.nsecs elapsed_time: 531.955219172 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 2 4 4 | 0 0 0 2: 1 2 6 | 0 0 0 3: 1 2 8 | 0 0 0 4: 1 2 10 | 0 0 0 5: 1 2 12 | 0 0 0 6: 1 2 14 | 0 0 0 7: 1 2 16 | 0 0 0 8: 1 2 18 | 0 0 0 9: 41 82 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff88012e148000.rpc_stats= snapshot_time: 1746141862.231459181 secs.nsecs start_time: 1746141328.515167848 secs.nsecs elapsed_time: 533.716291333 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1337 7 7 | 3 0 0 2: 9 0 7 | 0 0 0 4: 5 0 7 | 0 0 0 8: 15 0 7 | 0 0 0 16: 3769 21 29 | 0 0 0 32: 456 2 32 | 0 0 0 64: 480 2 35 | 0 0 0 128: 1290 7 42 | 0 0 0 256: 9861 57 100 | 2400 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 10353 60 60 | 266 11 11 2: 1676 9 69 | 584 24 35 3: 756 4 74 | 502 20 56 4: 577 3 77 | 365 15 71 5: 540 3 80 | 237 9 81 6: 443 2 83 | 148 6 87 7: 387 2 85 | 85 3 91 8: 383 2 87 | 81 3 94 9: 1841 10 98 | 129 5 99 10: 27 0 98 | 6 0 100 11: 20 0 98 | 0 0 100 12: 17 0 98 | 0 0 100 13: 16 0 98 | 0 0 100 14: 15 0 99 | 0 0 100 15: 13 0 99 | 0 0 100 16: 13 0 99 | 0 0 100 17: 12 0 99 | 0 0 100 18: 12 0 99 | 0 0 100 19: 12 0 99 | 0 0 100 20: 12 0 99 | 0 0 100 21: 12 0 99 | 0 0 100 22: 11 0 99 | 0 0 100 23: 8 0 99 | 0 0 100 24: 8 0 99 | 0 0 100 25: 8 0 99 | 0 0 100 26: 8 0 99 | 0 0 100 27: 8 0 99 | 0 0 100 28: 7 0 99 | 0 0 100 29: 5 0 99 | 0 0 100 30: 5 0 99 | 0 0 100 31: 17 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 197 1 1 | 51 2 2 1: 79 0 1 | 0 0 2 2: 149 0 2 | 0 0 2 4: 6 0 2 | 0 0 2 8: 1 0 2 | 0 0 2 16: 148 0 3 | 0 0 2 32: 266 1 4 | 0 0 2 64: 459 2 7 | 0 0 2 128: 807 4 12 | 0 0 2 256: 373 2 14 | 48 1 4 512: 662 3 18 | 96 3 8 1024: 1184 6 25 | 192 7 16 2048: 2636 15 40 | 384 15 32 4096: 4880 28 68 | 768 31 64 8192: 5375 31 100 | 864 35 100 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99.sanity-pcc is idle now PASS 99 (67s) == sanity-pcc test 99b: race among unlink | two readers | write | detach for PCC-RO file ========================================================== 19:24:25 (1746141865) 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.387625 s, 541 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: 1edd9147-430c-4bef-96e6-55e62f472f74 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f99b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1edd9147-430c-4bef-96e6-55e62f472f74 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.539218 s, 97.2 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.930735 s, 56.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00789 s, 52.0 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512819 s, 1.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.425978 s, 123 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.34839 s, 150 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.70036 s, 74.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.78485 s, 66.8 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.349935 s, 150 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.702301 s, 74.7 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.36528 s, 38.4 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.770124 s, 68.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.878527 s, 59.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.573677 s, 91.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0890112 s, 589 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.442417 s, 119 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672479 s, 78.0 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05639 s, 930 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348416 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.739268 s, 70.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036994 s, 1.4 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.641165 s, 81.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.629915 s, 83.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.836182 s, 62.7 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.668437 s, 78.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.535751 s, 97.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391507 s, 1.3 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373061 s, 1.4 GB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.391916 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.643023 s, 81.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571708 s, 917 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048595 s, 1.1 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.664992 s, 78.8 MB/s 52428800 bytes (52 MB) copied, 0.334003 s, 157 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337815 s, 1.6 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.780349 s, 67.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.92734 s, 56.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.330631 s, 159 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.499918 s, 105 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory 52428800 bytes (52 MB) copied, 0.0872896 s, 601 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.894537 s, 58.6 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225864 s, 232 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352113 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430533 s, 1.2 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382977 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.631173 s, 83.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041801 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.758353 s, 69.1 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.581593 s, 90.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.593065 s, 88.4 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517548 s, 1.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237845 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271717 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.637261 s, 82.3 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.381617 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270952 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.699652 s, 74.9 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.432122 s, 121 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.658613 s, 79.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615136 s, 852 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.756969 s, 69.3 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.787086 s, 66.6 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.816491 s, 64.2 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406716 s, 1.3 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.140002 s, 374 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.544133 s, 96.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380964 s, 1.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.731036 s, 71.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.718752 s, 72.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329782 s, 1.6 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.28893 s, 181 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.471107 s, 111 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247713 s, 2.1 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.494508 s, 106 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.555058 s, 94.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.433556 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033838 s, 1.5 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.358576 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.516288 s, 102 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076943 s, 681 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.281058 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.73576 s, 71.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362253 s, 1.4 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.435758 s, 120 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.657656 s, 79.7 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.712704 s, 73.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.560756 s, 93.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030221 s, 1.7 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.383843 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.625288 s, 83.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283582 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278709 s, 1.9 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.466915 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.520717 s, 101 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311043 s, 1.7 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.440594 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.620485 s, 84.5 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281135 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.541086 s, 96.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.7063 s, 74.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.550667 s, 95.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215562 s, 2.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.00369 s, 52.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.01186 s, 51.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304695 s, 1.7 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.487819 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.558585 s, 93.9 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211332 s, 248 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.255267 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.59904 s, 87.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275214 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.399577 s, 131 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.557825 s, 94.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237713 s, 2.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.649263 s, 80.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.590472 s, 88.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293744 s, 1.8 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.769896 s, 68.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.674568 s, 77.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209027 s, 2.5 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.499926 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.51775 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200812 s, 2.6 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.32774 s, 160 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.252074 s, 208 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216614 s, 242 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.459496 s, 114 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.305427 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320784 s, 1.6 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230513 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.544292 s, 96.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111175 s, 472 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.544777 s, 96.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294261 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.679332 s, 77.2 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.642379 s, 81.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.595878 s, 88.0 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389587 s, 1.3 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.389035 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.383602 s, 137 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207397 s, 253 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.504634 s, 104 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.567098 s, 92.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.621477 s, 84.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427249 s, 1.2 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.61608 s, 85.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.735804 s, 71.3 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354592 s, 1.5 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.363449 s, 144 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.451432 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.595933 s, 88.0 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314763 s, 1.7 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.270935 s, 194 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.692283 s, 75.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846802 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263819 s, 2.0 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.224448 s, 234 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.513913 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243007 s, 2.2 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59c6/0x2000059f1:0x37:0x0 ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.579175 s, 90.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.604317 s, 86.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.727159 s, 72.1 MB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff88012b636000.rpc_stats= snapshot_time: 1746141929.144920319 secs.nsecs start_time: 1746141330.275493023 secs.nsecs elapsed_time: 598.869427296 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 2 4 4 | 0 0 0 2: 1 2 6 | 0 0 0 3: 1 2 8 | 0 0 0 4: 1 2 10 | 0 0 0 5: 1 2 12 | 0 0 0 6: 1 2 14 | 0 0 0 7: 1 2 16 | 0 0 0 8: 2 4 20 | 0 0 0 9: 40 80 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0000-osc-ffff88012e148000.rpc_stats= snapshot_time: 1746141929.145062099 secs.nsecs start_time: 1746141328.510253123 secs.nsecs elapsed_time: 600.634808976 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 0 0 0 | 0 0 1 32: 0 0 0 | 0 0 1 64: 0 0 0 | 0 0 1 128: 0 0 0 | 0 0 1 256: 352 99 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 36 10 10 | 9 5 5 2: 27 7 17 | 37 24 30 3: 18 5 22 | 47 30 60 4: 18 5 27 | 27 17 78 5: 13 3 31 | 11 7 85 6: 13 3 35 | 11 7 92 7: 13 3 38 | 10 6 99 8: 22 6 45 | 1 0 100 9: 195 54 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 28 7 17 | 12 7 17 2048: 56 15 32 | 24 15 33 4096: 112 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff88012b636000.rpc_stats= snapshot_time: 1746141929.145130767 secs.nsecs start_time: 1746141330.276184014 secs.nsecs elapsed_time: 598.868946753 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 2 4 4 | 0 0 0 2: 1 2 6 | 0 0 0 3: 1 2 8 | 0 0 0 4: 1 2 10 | 0 0 0 5: 1 2 12 | 0 0 0 6: 1 2 14 | 0 0 0 7: 1 2 16 | 0 0 0 8: 1 2 18 | 0 0 0 9: 41 82 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff88012e148000.rpc_stats= snapshot_time: 1746141929.145193927 secs.nsecs start_time: 1746141328.515167848 secs.nsecs elapsed_time: 600.630026079 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1337 6 6 | 3 0 0 2: 9 0 6 | 0 0 0 4: 5 0 6 | 0 0 0 8: 15 0 6 | 0 0 0 16: 3769 18 24 | 0 0 0 32: 456 2 26 | 0 0 0 64: 480 2 29 | 0 0 0 128: 1290 6 35 | 0 0 0 256: 13382 64 100 | 5150 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 10912 52 52 | 467 9 9 2: 2062 9 62 | 1210 23 32 3: 1053 5 67 | 1163 22 55 4: 917 4 72 | 786 15 70 5: 876 4 76 | 591 11 81 6: 723 3 79 | 355 6 88 7: 637 3 82 | 218 4 92 8: 632 3 85 | 146 2 95 9: 2661 12 98 | 208 4 99 10: 31 0 98 | 9 0 100 11: 20 0 98 | 0 0 100 12: 17 0 99 | 0 0 100 13: 16 0 99 | 0 0 100 14: 15 0 99 | 0 0 100 15: 13 0 99 | 0 0 100 16: 13 0 99 | 0 0 100 17: 12 0 99 | 0 0 100 18: 12 0 99 | 0 0 100 19: 12 0 99 | 0 0 100 20: 12 0 99 | 0 0 100 21: 12 0 99 | 0 0 100 22: 11 0 99 | 0 0 100 23: 8 0 99 | 0 0 100 24: 8 0 99 | 0 0 100 25: 8 0 99 | 0 0 100 26: 8 0 99 | 0 0 100 27: 8 0 99 | 0 0 100 28: 7 0 99 | 0 0 100 29: 5 0 99 | 0 0 100 30: 5 0 99 | 0 0 100 31: 17 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 274 1 1 | 106 2 2 1: 79 0 1 | 0 0 2 2: 149 0 2 | 0 0 2 4: 6 0 2 | 0 0 2 8: 1 0 2 | 0 0 2 16: 148 0 3 | 0 0 2 32: 266 1 4 | 0 0 2 64: 459 2 6 | 0 0 2 128: 807 3 10 | 0 0 2 256: 445 2 12 | 103 1 4 512: 802 3 16 | 206 3 8 1024: 1464 7 23 | 412 7 16 2048: 3192 15 39 | 824 15 32 4096: 6009 28 67 | 1648 31 64 8192: 6642 32 100 | 1854 35 100 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99b.sanity-pcc is idle now PASS 99b (67s) == sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 19:25:32 (1746141932) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.112072 s, 561 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 01b2edaf-b958-480a-a0a8-931e258dfa37 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=01b2edaf-b958-480a-a0a8-931e258dfa37 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.480251 s, 109 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0545074 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0047815 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.529824 s, 99.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.250311 s, 209 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 9+0 records in 9+0 records out 9437184 bytes (9.4 MB) copied, 0.0142599 s, 662 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.556755 s, 94.2 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543399 s, 96.5 MB/s 39+0 records in 39+0 records out 40894464 bytes (41 MB) copied, 0.165649 s, 247 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.747193 s, 70.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.353607 s, 148 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0509414 s, 20.6 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0216545 s, 48.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.675883 s, 77.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.38599 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267215 s, 2.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.677471 s, 77.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.372279 s, 141 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.051264 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00700084 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.571022 s, 91.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232604 s, 225 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255408 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326097 s, 1.6 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00340175 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00211632 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0072633 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.476048 s, 110 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0075598 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.131629 s, 215 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0436264 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.378401 s, 139 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143029 s, 367 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.812151 s, 64.6 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0317428 s, 33.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.655694 s, 80.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.428879 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602522 s, 870 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 16+0 records in 16+0 records out 16777216 bytes (17 MB) copied, 0.0209859 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.669112 s, 78.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.435002 s, 121 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.165027 s, 146 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.157926 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.678366 s, 77.3 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.692024 s, 75.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137088 s, 382 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.66004 s, 79.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.403453 s, 130 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0551656 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.778927 s, 67.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.459633 s, 114 MB/s 9+0 records in 9+0 records out 9437184 bytes (9.4 MB) copied, 0.0276327 s, 342 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.586311 s, 89.4 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.195929 s, 53.5 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0242892 s, 43.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.728447 s, 72.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.482075 s, 109 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.190529 s, 253 MB/s 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.0596951 s, 105 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.44174 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.645266 s, 81.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.20177 s, 43.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0070813 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0136525 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 dd: warning: partial read (290816 bytes); suggest iflag=fullblock 46+3 records in 46+3 records out 49283072 bytes (49 MB) copied, 0.403699 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.748944 s, 70.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351171 s, 1.5 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.539507 s, 97.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291851 s, 1.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 26+0 records in 26+0 records out 27262976 bytes (27 MB) copied, 0.157163 s, 173 MB/s 44+0 records in 44+0 records out 46137344 bytes (46 MB) copied, 0.171655 s, 269 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.734608 s, 71.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.538129 s, 97.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415198 s, 126 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0208369 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.626198 s, 83.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.545094 s, 96.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311839 s, 1.7 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.711391 s, 73.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.575549 s, 91.1 MB/s 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.305981 s, 123 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00774816 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.809028 s, 64.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.428888 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.562284 s, 93.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.412735 s, 5.1 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0297897 s, 35.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.64136 s, 81.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.301364 s, 174 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289773 s, 1.8 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 11+0 records in 11+0 records out 11534336 bytes (12 MB) copied, 0.0241702 s, 477 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.708915 s, 74.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.42457 s, 123 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362312 s, 1.4 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.188916 s, 128 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.784588 s, 66.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312503 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394495 s, 1.3 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00646644 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00310576 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.210116 s, 160 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.795986 s, 65.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301581 s, 1.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 28+0 records in 28+0 records out 29360128 bytes (29 MB) copied, 0.205386 s, 143 MB/s 13+0 records in 13+0 records out 13631488 bytes (14 MB) copied, 0.192987 s, 70.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.721509 s, 72.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 26+0 records in 26+0 records out 27262976 bytes (27 MB) copied, 0.668768 s, 40.8 MB/s 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.352859 s, 95.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.591688 s, 88.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000625541 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0474544 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.661866 s, 79.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.218348 s, 240 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0381491 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0256155 s, 287 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.46907 s, 112 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0149005 s, 0.0 kB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0175625 s, 59.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.385047 s, 136 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 39+0 records in 39+0 records out 40894464 bytes (41 MB) copied, 0.124276 s, 329 MB/s 18+0 records in 18+0 records out 18874368 bytes (19 MB) copied, 0.118103 s, 160 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.706959 s, 74.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410162 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393533 s, 1.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.261887 s, 200 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.490096 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.784161 s, 66.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0367288 s, 0.0 kB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0418751 s, 25.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.742765 s, 70.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.421204 s, 124 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.497589 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366805 s, 1.4 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 39+0 records in 39+0 records out 40894464 bytes (41 MB) copied, 0.341449 s, 120 MB/s 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.175274 s, 239 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.774623 s, 67.7 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0595494 s, 17.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 13+0 records in 13+0 records out 13631488 bytes (14 MB) copied, 0.0353012 s, 386 MB/s 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.378383 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.937683 s, 55.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108807 s, 482 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885908 s, 592 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0181912 s, 0.0 kB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.0241492 s, 651 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.904899 s, 57.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0501054 s, 20.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0240676 s, 43.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.564931 s, 92.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.396107 s, 132 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407583 s, 1.3 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.548691 s, 93.6 MB/s 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.379281 s, 135 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.999893 s, 52.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0367192 s, 28.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 37+0 records in 37+0 records out 38797312 bytes (39 MB) copied, 0.0584937 s, 663 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.784364 s, 66.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 8+0 records in 8+0 records out 8388608 bytes (8.4 MB) copied, 0.24076 s, 34.8 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0144143 s, 72.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.505437 s, 104 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.32588 s, 161 MB/s 32+0 records in 32+0 records out 33554432 bytes (34 MB) copied, 0.131914 s, 254 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.461268 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.443176 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0980758 s, 535 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0187469 s, 112 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.551857 s, 95.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.511064 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025807 s, 2.0 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.728518 s, 72.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.313212 s, 167 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409076 s, 128 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.44461 s, 23.6 MB/s 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0243051 s, 216 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59a9/0x2000059f1:0x58:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.544655 s, 96.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.561057 s, 93.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244148 s, 2.1 GB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now PASS 100 (65s) == sanity-pcc test 101a: Test auto attach in mount namespace (simulated container) ========================================================== 19:26:37 (1746141997) creating user namespace for 500 sanityu+ 18408 0.0 0.0 108060 368 pts/0 S+ 19:26 0:00 sleep 600 root 18410 0.0 0.0 114880 984 pts/0 S+ 19:26 0:00 grep sleep Created NS: child (sleep) pid=18408 in 0 seconds running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newuidmap] [18408] [0] [500] [1] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newgidmap] [18408] [0] [500] [1] 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764134 s, 686 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d101a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 06ad53d6-b608-403e-b713-1ae4d0141c25 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f101a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=06ad53d6-b608-403e-b713-1ae4d0141c25 (extents) (64bit) (large files) (huge files) Check for temp file in PCC mount Check for temp file in PCC mount from inside namespace Starting copytool 'agt1' on 'oleg252-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d101a.sanity-pcc/d101a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach from inside mount namespace 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00940459 s, 109 kB/s file: /mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc, type: readwrite, PCC_file: /d101a.sanity-pcc/5982/0x2000059f1:0x73:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012e148000.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00126434 s, 810 kB/s Verify auto attach at open from inside NS for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff88012b636000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012e148000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d101a.sanity-pcc is idle now PASS 101a (9s) == sanity-pcc test 102: PCC-RO should not hange for io_uring I/O engine ========================================================== 19:26:46 (1746142006) SKIP: sanity-pcc test_102 Client OS does not support io_uring I/O engine SKIP 102 (2s) == sanity-pcc test 203: Verify attach/hit bytes statistics data ========================================================== 19:26:48 (1746142008) 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0378068 s, 277 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d203.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/10240 done Creating filesystem with 10240 1k blocks and 2560 inodes Filesystem UUID: 817e0fc8-611c-4cde-940b-39d72e12699a Superblock backups stored on blocks: 8193 Allocating group tables: 0/2 done Writing inode tables: 0/2 done Creating journal (2048 blocks): done Writing superblocks and filesystem accounting information: 0/2 done /tmp/f203.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=817e0fc8-611c-4cde-940b-39d72e12699a (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d203.sanity-pcc/d203.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00914549 s, 112 kB/s ==== stats ==== llite.lustre-ffff88012b636000.stats= snapshot_time 1746142009.934420137 secs.nsecs start_time 1746142009.832273649 secs.nsecs elapsed_time 0.102146488 secs.nsecs llite.lustre-ffff88012e148000.stats= snapshot_time 1746142009.936426485 secs.nsecs start_time 1746142009.832293260 secs.nsecs elapsed_time 0.104133225 secs.nsecs read_bytes 1 samples [bytes] 1024 1024 1024 1048576 write_bytes 1 samples [bytes] 1024 1024 1024 1048576 read 3 samples [usecs] 187 8433 10050 73195358 write 1 samples [usecs] 2962 2962 2962 8773444 open 3 samples [usecs] 17 23828 23868 567774402 close 3 samples [usecs] 2232 5783 10535 44775313 getattr 2 samples [usecs] 67 2446 2513 5987405 mknod 1 samples [usecs] 15703 15703 15703 246584209 inode_permission 7 samples [usecs] 2 5671 5938 32185786 opencount 3 samples [reqs] 1 3 6 14 openclosetime 2 samples [usecs] 13789 18847 32636 545345930 pcc_attach 1 samples [reqs] pcc_hit_bytes 2 samples [bytes] 1024 1024 2048 2097152 pcc_attach_bytes 1 samples [bytes] 1024 1024 1024 1048576 attach_num: 1, attach_bytes: 1024, hit_num: 2, hit_bytes: 2048 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d203.sanity-pcc is idle now PASS 203 (4s) == sanity-pcc test 204a: pin/unpin pcc flag ============== 19:26:53 (1746142013) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843686 s, 621 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 30c9e424-83b4-4c5b-b86b-1647caa09a41 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f204a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=30c9e424-83b4-4c5b-b86b-1647caa09a41 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204a.sanity-pcc is idle now PASS 204a (6s) == sanity-pcc test 204b: Permission check for the pin/unpin operation ========================================================== 19:26:58 (1746142018) mdt.lustre-MDT0000.enable_pin_gid=0 /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh: illegal option -- p running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc trusted.pin="[hsm: 100]" running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] unpin: cannot unpin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted mdt.lustre-MDT0000.enable_pin_gid=-1 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=500 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=0 PASS 204b (7s) == sanity-pcc test 204c: PCC pin/unpin without attach ID specified ========================================================== 19:27:05 (1746142025) pin: failed to get id for '/mnt/lustre/f204c.sanity-pcc': No such file or directory 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.109805 s, 573 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12755) mqueue on /dev/mqueue type mqueue (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg252-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg252-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51286/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.202.52,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.202.152@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.202.152@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: f1fa24c7-59ef-4cb3-bcfc-890942760049 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f204c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=f1fa24c7-59ef-4cb3-bcfc-890942760049 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204c.sanity-pcc trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204c.sanity-pcc is idle now PASS 204c (4s) == sanity-pcc test complete, duration 1957 sec =========== 19:27:09 (1746142029) sanity-pcc: FAIL: test_1c expected /mnt/pcc.d1c.sanity-pcc/d1c.sanity-pcc/0bd3/0x200000bd1:0x2:0x0 size: 7654321 got: 0 === sanity-pcc: start cleanup 19:27:10 (1746142030) === Stopping clients: oleg252-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg252-client.virtnet /mnt/lustre2 opts: === sanity-pcc: finish cleanup 19:27:14 (1746142034) ===