-----============= acceptance-small: sanity-pcc ============----- Thu May 1 19:26:04 EDT 2025 mgs: CentOS Linux release 7.9.2009 (Core) MGS_OS_VERSION_ID=7 MGS_OS_ID=centos MGS_OS_VERSION_CODE=117440512 MGS_OS_ID_LIKE=rhel fedora centos mds1: CentOS Linux release 7.9.2009 (Core) MDS1_OS_ID_LIKE=rhel fedora centos MDS1_OS_ID=centos MDS1_OS_VERSION_ID=7 MDS1_OS_VERSION_CODE=117440512 ost1: CentOS Linux release 7.9.2009 (Core) OST1_OS_VERSION_CODE=117440512 OST1_OS_VERSION_ID=7 OST1_OS_ID_LIKE=rhel fedora centos OST1_OS_ID=centos client: CentOS Linux release 7.9.2009 (Core) CLIENT_OS_ID=centos CLIENT_OS_ID_LIKE=rhel fedora centos CLIENT_OS_VERSION_ID=7 CLIENT_OS_VERSION_CODE=117440512 === sanity-pcc: start setup 19:26:11 (1746141971) === Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) oleg413-client.virtnet: executing check_config_client /mnt/lustre oleg413-client.virtnet: Checking config lustre mounted on /mnt/lustre Checking servers environments Checking clients oleg413-client.virtnet environments Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800af4f5800.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012b73d000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800af4f5800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012b73d000.idle_timeout=debug disable quota as required oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all === sanity-pcc: finish setup 19:26:26 (1746141986) === running as uid/gid/euid/egid 500/500/500/500, groups: 500 [true] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d0_runas_test/f7025] nobody:x:99: oleg413-server: ls: cannot access /home/green/git/lustre-release/lustre/tests/except/sanity-pcc.*ex: No such file or directory excepting tests: Killing existing copytools on oleg413-client.virtnet pkill: pidfile not valid Try `pkill --help' for more information. Set HSM on and start Set sanity-hsm HSM policy == sanity-pcc test 1a: Test manual lfs pcc attach with manual HSM restore ========================================================== 19:26:50 (1746142010) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966735 s, 542 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b18dcf34-117e-48cf-a494-820b5782def2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b18dcf34-117e-48cf-a494-820b5782def2 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1a.sanity-pcc/d1a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0519795 s, 147 MB/s Restore testing... Waiting 200s for 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1a.sanity-pcc/f1a.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1a.sanity-pcc is idle now PASS 1a (10s) == sanity-pcc test 1b: Test manual lfs pcc attach with restore on remote access ========================================================== 19:27:00 (1746142020) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0851437 s, 616 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: adf52b48-a783-4e4e-a0d5-4b2e3f73f833 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=adf52b48-a783-4e4e-a0d5-4b2e3f73f833 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1b.sanity-pcc/d1b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' using SAVE_PROJECT_SUPPORTED=0 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0610874 s, 125 MB/s Restore testing... Waiting 200s for 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1b.sanity-pcc/f1b.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1b.sanity-pcc is idle now PASS 1b (13s) == sanity-pcc test 1c: Test automated attach using Project ID with manual HSM restore ========================================================== 19:27:13 (1746142033) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions oleg413-server: libkmod: kmod_module_get_refcnt: could not read integer from '/sys/module/acpi_cpufreq/refcnt': 'No such device' oleg413-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory oleg413-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800aadf4000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012b47a000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aadf4000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012b47a000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716856 s, 731 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2cfed277-916d-403f-981d-6c6e7df35e75 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2cfed277-916d-403f-981d-6c6e7df35e75 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1c.sanity-pcc/d1c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0498343 s, 154 MB/s Restore testing... Waiting 200s for 'SUCCEED' Attach and detach testing Start to detach the /mnt/lustre/d1c.sanity-pcc/f1c.sanity-pcc Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1c.sanity-pcc is idle now PASS 1c (72s) == sanity-pcc test 1d: Test Project ID with remote access ========================================================== 19:28:25 (1746142105) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions oleg413-server: libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800abee7800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800af51b000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800abee7800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800af51b000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571586 s, 917 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e2e0c1e6-761d-4f74-922b-6c62de4fe055 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e2e0c1e6-761d-4f74-922b-6c62de4fe055 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1d.sanity-pcc/d1d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 7654321 bytes (7.7 MB) copied, 0.0952057 s, 80.4 MB/s Restore testing... Attach and detach testing Start to detach the /mnt/lustre/d1d.sanity-pcc/f1d.sanity-pcc Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1d.sanity-pcc is idle now PASS 1d (70s) == sanity-pcc test 1e: Test RW-PCC with non-root user ==== 19:29:35 (1746142175) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_1e/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_1e/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00598315 s, 171 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00325823 s, 314 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00395482 s, 259 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] touch: cannot touch '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0] [bs=1024] [count=1] dd: failed to open '/tmp/arc1/sanity-pcc.test_1e//13a6/0x2000013a1:0x7:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1e.sanity-pcc/f1e.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre PASS 1e (7s) == sanity-pcc test 1f: Test auto RW-PCC cache with non-root user ========================================================== 19:29:42 (1746142182) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b6d03000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012b73f800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b6d03000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012b73f800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598929 s, 875 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 828cdb9e-6373-41e6-a7b5-8fb8e12ff12c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=828cdb9e-6373-41e6-a7b5-8fb8e12ff12c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00426978 s, 240 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00100451 s, 1.0 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] [bs=256] [count=1] 1+0 records in 1+0 records out 256 bytes (256 B) copied, 0.00410209 s, 62.4 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] touch: cannot touch '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d1f.sanity-pcc/d1f.sanity-pcc/1b73/0x200001b71:0x2:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d1f.sanity-pcc/f1f.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1f.sanity-pcc is idle now PASS 1f (49s) == sanity-pcc test 1g: General permission test for RW-PCC ========================================================== 19:30:31 (1746142231) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681116 s, 770 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d1g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c09eecf2-2f52-44b3-91e5-9cc31dc8d9f2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f1g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c09eecf2-2f52-44b3-91e5-9cc31dc8d9f2 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d1g.sanity-pcc/d1g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00585719 s, 175 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/f1g.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/f1g.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00311069 s, 329 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] detach: cannot detach '/mnt/lustre/f1g.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f1g.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f1g.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.012954 s, 79.0 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d1g.sanity-pcc is idle now PASS 1g (7s) == sanity-pcc test 2a: Test multi open when creating ===== 19:30:38 (1746142238) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions oleg413-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff88012b73e000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012dc56800.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012b73e000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012dc56800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770879 s, 680 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 84bfb44d-784f-4acc-a735-b9c185cf2dc0 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=84bfb44d-784f-4acc-a735-b9c185cf2dc0 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2a.sanity-pcc/multiop vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.24626 client oleg413-client.virtnet multiop_bg started multiop_pid=24636 ---------- 1 root root 14 May 1 19:31 /mnt/pcc.d2a.sanity-pcc/d2a.sanity-pcc/2343/0x200002341:0x2:0x0 Stopping multiop_pid=24636 (kill 24636 on oleg413-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d2a.sanity-pcc is idle now PASS 2a (66s) == sanity-pcc test 2b: Test multi remote open when creating ========================================================== 19:31:44 (1746142304) SKIP: sanity-pcc test_2b Need 2 or more clients, have 1 SKIP 2b (1s) == sanity-pcc test 2c: Test multi open on different mount points when creating ========================================================== 19:31:45 (1746142305) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions oleg413-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800aae88800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800af4f1800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aae88800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800af4f1800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623679 s, 841 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d2c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 66e0d8d9-e895-48c6-a892-184e491929bd Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f2c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=66e0d8d9-e895-48c6-a892-184e491929bd (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d2c.sanity-pcc/d2c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/d2c.sanity-pcc/f2c.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.27800 client oleg413-client.virtnet multiop_bg started multiop_pid=27810 Stopping multiop_pid=27810 (kill 27810 on oleg413-client.virtnet) multiopen_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d2c.sanity-pcc is idle now PASS 2c (49s) == sanity-pcc test 3a: Repeat attach/detach operations === 19:32:34 (1746142354) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_3a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00766034 s, 134 kB/s Start to RW-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RW-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Start to RO-PCC attach/detach the file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Repeat to RO-PCC attach/detach the same file: /mnt/lustre/d3a.sanity-pcc/f3a.sanity-pcc Cleanup PCC backend on /mnt/lustre PASS 3a (5s) == sanity-pcc test 3b: Repeat attach/detach operations on multiple clients ========================================================== 19:32:39 (1746142359) SKIP: sanity-pcc test_3b Need 3 or more clients, have 1 SKIP 3b (1s) == sanity-pcc test 4: Auto cache test for mmap =========== 19:32:40 (1746142360) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b5831000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88012a66c800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b5831000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88012a66c800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667344 s, 786 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d4.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e45fa7ce-4f9e-432a-99ec-1e8e57eabeb9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f4.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e45fa7ce-4f9e-432a-99ec-1e8e57eabeb9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d4.sanity-pcc/d4.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' mmap test1: basic mmap operation (PASS, 0.016068s) mmap test2: MAP_PRIVATE not write back (PASS, 0.040145s) mmap test3: concurrent mmap ops on two nodes (PASS, 2.0261s) mmap test4: c1 write to f1 from mmapped f2, c2 write to f1 from mmapped f1 (PASS, 4.077s) mmap test5: read/write file to/from the buffer which mmapped to just this file (PASS, 0.028751s) mmap test6: check mmap write/read content on two nodes (PASS, 1.2019s) mmap test7: file i/o with an unmapped buffer (SKIPPED, 0s) mmap test8: SIGBUS for beyond file size (SKIPPED, 0s) mmap test9: SIGBUS for negative file offset (SKIPPED, 0s) mmap test10: multi-thread mmap access (PASS, 0.14035s) mmap test11: multi-thread shared mmap (PASS, 0.15085s) mmap test10: mtime not change for readonly mmap access (PASS, 1.0161s) ldlm.namespaces.lustre-MDT0000-mdc-ffff8800b5831000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff88012a66c800.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d4.sanity-pcc is idle now PASS 4 (76s) == sanity-pcc test 5: Mmap & cat a RW-PCC cached file ==== 19:33:57 (1746142437) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758116 s, 692 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d5.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 15072664-ed40-4c18-a5a0-5f56c12b9300 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f5.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=15072664-ed40-4c18-a5a0-5f56c12b9300 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d5.sanity-pcc/d5.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d5.sanity-pcc is idle now PASS 5 (6s) == sanity-pcc test 6: Test mmap write on RW-PCC ========== 19:34:03 (1746142443) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715182 s, 733 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d6.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 06af4c0a-f5c4-4032-93db-659b3896fd5b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f6.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=06af4c0a-f5c4-4032-93db-659b3896fd5b (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d6.sanity-pcc/d6.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d6.sanity-pcc is idle now PASS 6 (7s) == sanity-pcc test 7a: Fake file detached between fault() and page_mkwrite() for RW-PCC ========================================================== 19:34:10 (1746142450) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0877448 s, 598 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e7a8c2fe-0db3-4bae-ac71-61a1f74fe968 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e7a8c2fe-0db3-4bae-ac71-61a1f74fe968 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7a.sanity-pcc/d7a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1412 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7a.sanity-pcc is idle now PASS 7a (5s) == sanity-pcc test 7b: Test the race with concurrent mkwrite and detach ========================================================== 19:34:15 (1746142455) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0780623 s, 672 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d7b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: eb766d94-80d3-4564-8cd4-065d462f8747 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f7b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=eb766d94-80d3-4564-8cd4-065d462f8747 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d7b.sanity-pcc/d7b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1413 fail_val=20 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d7b.sanity-pcc is idle now PASS 7b (45s) == sanity-pcc test 8: Test fake -ENOSPC tolerance for RW-PCC ========================================================== 19:35:00 (1746142500) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0745864 s, 703 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d8.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 239ed44a-7ae0-4d62-9cc6-e809e6ac21c5 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f8.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=239ed44a-7ae0-4d62-9cc6-e809e6ac21c5 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d8.sanity-pcc/d8.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d8.sanity-pcc is idle now PASS 8 (6s) == sanity-pcc test 9: Test -ENOSPC tolerance on loop PCC device for RW-PCC ========================================================== 19:35:06 (1746142506) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607261 s, 863 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.9a: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2934b6d3-3617-430b-a623-2d1e1b62d3da Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f9.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2934b6d3-3617-430b-a623-2d1e1b62d3da (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.9a/d9.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.9a/d9.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 1.57252 s, 40.0 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.9a is idle now PASS 9 (6s) == sanity-pcc test 10a: Test RW-PCC with user quota on loop PCC device ========================================================== 19:35:12 (1746142512) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.089856 s, 583 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 63a33183-a6b7-441f-ae95-be1b663675c7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=63a33183-a6b7-441f-ae95-be1b663675c7 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10a.sanity-pcc/d10a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.161636 s, 97.3 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.168568 s, 93.3 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10a.sanity-pcc/uquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x26:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10a.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10a.sanity-pcc/uquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 0.898172 s, 35.0 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10a.sanity-pcc is idle now PASS 10a (6s) == sanity-pcc test 10b: Test RW-PCC with group quota on loop PCC device ========================================================== 19:35:18 (1746142518) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974654 s, 538 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: cd7cb025-cfb0-454d-9d11-d0fcdc1fa20c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=cd7cb025-cfb0-454d-9d11-d0fcdc1fa20c (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10b.sanity-pcc/d10b.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.137311 s, 115 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.156471 s, 101 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [-w] [/mnt/lustre/d10b.sanity-pcc/gquotaB] lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x2b:0x0]: Disk quota exceeded (122) lfs pcc pcc: cannot attach with ID: 2: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10b.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Test -EDQUOT error tolerance for RW-PCC running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d10b.sanity-pcc/gquotaA] [bs=1M] [count=30] 30+0 records in 30+0 records out 31457280 bytes (31 MB) copied, 1.15176 s, 27.3 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10b.sanity-pcc is idle now PASS 10b (6s) == sanity-pcc test 10c: Test RO-PCC with user quota on loop PCC device ========================================================== 19:35:24 (1746142524) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0887835 s, 591 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 2c22ef52-2be3-41e4-9d84-98e59488ad4f Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=2c22ef52-2be3-41e4-9d84-98e59488ad4f (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10c.sanity-pcc/d10c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.133852 s, 118 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.12169 s, 129 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10c.sanity-pcc/uquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10c.sanity-pcc/uquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10c.sanity-pcc is idle now PASS 10c (5s) == sanity-pcc test 10d: Test RO-PCC with group quota on loop PCC device ========================================================== 19:35:29 (1746142529) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080234 s, 653 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 45628a56-12af-4256-9bb3-f343a97f7039 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=45628a56-12af-4256-9bb3-f343a97f7039 (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 20480K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d10d.sanity-pcc/d10d.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.156459 s, 101 MB/s 15+0 records in 15+0 records out 15728640 bytes (16 MB) copied, 0.170146 s, 92.4 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaA] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-i] [2] [/mnt/lustre/d10d.sanity-pcc/gquotaB] lfs pcc pcc: cannot attach the file to PCC with ID 2 failed: Disk quota exceeded (122) attach: cannot attach '/mnt/lustre/d10d.sanity-pcc/gquotaB' to PCC with attach ID '2': Disk quota exceeded Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10d.sanity-pcc is idle now PASS 10d (5s) == sanity-pcc test 10e: Tolerate -EDQUOT failure when auto PCC-RO attach with user quota ========================================================== 19:35:34 (1746142534) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0852589 s, 615 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d430bff1-a7d2-4845-b1f9-3de602ecd254 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d430bff1-a7d2-4845-b1f9-3de602ecd254 (extents) (64bit) (large files) (huge files) *** Report for user quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits User used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10e.sanity-pcc/d10e.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: uid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0452048 s, 46.4 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0111221 s, 189 MB/s /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0534849 s, 98.0 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10e.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.031556 s, 166 MB/s file: /mnt/lustre/f10e.sanity-pcc, type: none /mnt/lustre/f10e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x34 lmm_fid: [0x2000032e1:0x34:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10e.sanity-pcc is idle now PASS 10e (5s) == sanity-pcc test 10f: Tolerate -EDQUOT failure when auto PCC-RO attach with group quota ========================================================== 19:35:39 (1746142539) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101691 s, 516 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d10f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a1b33546-c5a0-4161-ba82-6a922291a86d Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f10f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a1b33546-c5a0-4161-ba82-6a922291a86d (extents) (64bit) (large files) (huge files) *** Report for group quotas on device /dev/loop0 Block grace time: 7days; Inode grace time: 7days Space limits File limits Group used soft hard grace used soft hard grace ---------------------------------------------------------------------- root -- 13K 0K 0K 2 0 0 sanityusr -- 0K 0K 4096K 0 0 0 Statistics: Total blocks: 7 Data blocks: 1 Entries: 2 Used average: 2.000000 pcc: - pccpath: /mnt/pcc.d10f.sanity-pcc/d10f.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: gid={500} 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0301431 s, 69.6 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=2] 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0105858 s, 198 MB/s /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0771901 s, 67.9 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/f10f.sanity-pcc] [of=/dev/null] [bs=1M] [count=5] 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0262504 s, 200 MB/s file: /mnt/lustre/f10f.sanity-pcc, type: none /mnt/lustre/f10f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000032e1 lmm_object_id: 0x36 lmm_fid: [0x2000032e1:0x36:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0xcf:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d10f.sanity-pcc is idle now PASS 10f (6s) == sanity-pcc test 11: Test attach fault injection with simulated PCC file path ========================================================== 19:35:45 (1746142545) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0740014 s, 708 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d11.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b8414918-4501-4d12-b811-d7adffdde0ed Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f11.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b8414918-4501-4d12-b811-d7adffdde0ed (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d11.sanity-pcc/d11.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32d9 Lustre file: /mnt/lustre/f11.sanity-pcc LPCC dir: /mnt/pcc.d11.sanity-pcc/d11.sanity-pcc/32db lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3a:0x0]: Permission denied (13) lfs pcc pcc: cannot attach with ID: 2: Permission denied (13) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Permission denied lfs pcc pcc: cannot get UNLOCK lease, ext 10 for [0x2000032e1:0x3b:0x0]: Invalid argument (22) lfs pcc pcc: cannot attach with ID: 2: Invalid argument (22) attach: cannot attach '/mnt/lustre/f11.sanity-pcc' to PCC with attach ID '2': Invalid argument Cleanup PCC backend on /mnt/lustre /mnt/pcc.d11.sanity-pcc is idle now PASS 11 (5s) == sanity-pcc test 12: RW-PCC attach races with concurrent HSM remove ========================================================== 19:35:50 (1746142550) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_12/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1414 fail_val=20 /mnt/lustre/f12.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre PASS 12 (25s) == sanity-pcc test 13a: Test auto RW-PCC create caching for UID/GID rule ========================================================== 19:36:15 (1746142575) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13a/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: uid={500} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00483426 s, 212 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.000808662 s, 1.3 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00446464 s, 229 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13a/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: gid={500} lfs mkdir: dirstripe error on '/mnt/lustre/d13a.sanity-pcc': stripe already set lfs setdirstripe: cannot create dir '/mnt/lustre/d13a.sanity-pcc': File exists running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00322162 s, 318 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.000973958 s, 1.1 MB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [256] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/tests/truncate] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [2048] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00467131 s, 219 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d13a.sanity-pcc/f13a.sanity-pcc] Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre PASS 13a (10s) == sanity-pcc test 13b: Test auto RW-PCC create caching for file name with wildcard ========================================================== 19:36:25 (1746142585) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_13b/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /tmp/arc1/sanity-pcc.test_13b/ hsmtool: posix_v2 rwid: 2 roid: 0 flags: 90 autocache: fname={*.h5 suffix.* Mid*dle} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00471486 s, 217 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/suffix.doc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00561542 s, 182 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/MidPADdle] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00202277 s, 506 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13b.sanity-pcc/Midpad] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00829365 s, 123 kB/s Cleanup PCC backend on /mnt/lustre PASS 13b (4s) == sanity-pcc test 13c: Check auto RW-PCC create caching for UID/GID/ProjID/fname rule ========================================================== 19:36:29 (1746142589) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions oleg413-server: libkmod: kmod_module_get_holders: could not open '/sys/module/intel_rapl/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800af518000.idle_timeout=debug osc.lustre-OST0000-osc-ffff880130f34000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800af518000.idle_timeout=debug osc.lustre-OST0001-osc-ffff880130f34000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820905 s, 639 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d13c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a772ad53-81ca-42c4-b0e0-6016846ddad2 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f13c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a772ad53-81ca-42c4-b0e0-6016846ddad2 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d13c.sanity-pcc/d13c.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100 200}&fname={*.h5},uid={500}&gid={500} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00923886 s, 111 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00260088 s, 394 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00509864 s, 201 kB/s 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00274782 s, 373 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d13c.sanity-pcc/ugidcache] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0020337 s, 504 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d13c.sanity-pcc is idle now PASS 13c (53s) == sanity-pcc test 14: Revocation of the layout lock should detach the file automatically ========================================================== 19:37:22 (1746142642) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/tmp/arc1/sanity-pcc.test_14/ --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear Cleanup PCC backend on /mnt/lustre PASS 14 (4s) == sanity-pcc test 15: Test auto attach at open when file is still valid cached ========================================================== 19:37:26 (1746142646) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0842355 s, 622 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d15.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1910c2ae-af37-47ef-9e02-b02de962b987 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f15.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1910c2ae-af37-47ef-9e02-b02de962b987 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d15.sanity-pcc/d15.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach for non-root user running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0120051 s, 85.3 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-w] [-i] [2] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] file: /mnt/lustre/d15.sanity-pcc/f15.sanity-pcc, type: readwrite, PCC_file: /d15.sanity-pcc/3aa0/0x200003ab1:0x11:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d15.sanity-pcc/f15.sanity-pcc] Verify auto attach at open for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear Waiting 200s for 'SUCCEED' Verify auto attach at open for RO-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear Cleanup PCC backend on /mnt/lustre /mnt/pcc.d15.sanity-pcc is idle now PASS 15 (8s) == sanity-pcc test 16: Test detach with different options ========================================================== 19:37:34 (1746142654) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659002 s, 796 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d16.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: cd3ee484-cead-4630-aa15-7c5773bc94a7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f16.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=cd3ee484-cead-4630-aa15-7c5773bc94a7 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d16.sanity-pcc/d16.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Test detach for RW-PCC Test for reusing valid PCC cache Test for the default detach Waiting 200s for 'SUCCEED' Test detach for RO-PCC detach_dataCleanup PCC backend on /mnt/lustre /mnt/pcc.d16.sanity-pcc is idle now PASS 16 (6s) == sanity-pcc test 17: Test auto attach for layout refresh ========================================================== 19:37:40 (1746142660) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067425 s, 778 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d17.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 80fd5c72-6cc9-4121-8578-bbfb5d1e2aba Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f17.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=80fd5c72-6cc9-4121-8578-bbfb5d1e2aba (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d17.sanity-pcc/d17.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d17.sanity-pcc/d17.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 94 autocache: projid={100} Verify auto attach during IO for truncate Verify auto attach during IO for read/write multiop /mnt/lustre/f17.sanity-pcc vO_r TMPPIPE=/tmp/multiop_open_wait_pipe.19263 client oleg413-client.virtnet multiop_bg started multiop_pid=19274 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear Stopping multiop_pid=19274 (kill 19274 on oleg413-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d17.sanity-pcc is idle now PASS 17 (13s) == sanity-pcc test 18: Verify size correctness after re-attach the file ========================================================== 19:37:53 (1746142673) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0818425 s, 641 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d18.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e0ecca1b-5630-4f04-860d-559a79bdaf60 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f18.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e0ecca1b-5630-4f04-860d-559a79bdaf60 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d18.sanity-pcc/d18.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d18.sanity-pcc/d18.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 0 flags: 9e autocache: projid={100} 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.10884 s, 38.5 MB/s file: /mnt/lustre/f18.sanity-pcc, type: readwrite, PCC_file: /d18.sanity-pcc/3aac/0x200003ab1:0x1d:0x0, open_count: 0, flags: 0 file: /mnt/lustre/f18.sanity-pcc, type: none /mnt/lustre/f18.sanity-pcc has size 4194304 OK 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00589282 s, 174 kB/s /mnt/lustre2/f18.sanity-pcc has size 1049600 OK file: /mnt/lustre/f18.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d18.sanity-pcc is idle now PASS 18 (6s) == sanity-pcc test 19: Verify the file re-attach works as expected ========================================================== 19:37:59 (1746142679) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0690915 s, 759 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d19.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c9ef85a5-28d0-46bb-aa62-c6a302321d95 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f19.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c9ef85a5-28d0-46bb-aa62-c6a302321d95 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d19.sanity-pcc/d19.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d19.sanity-pcc is idle now PASS 19 (5s) == sanity-pcc test 20: Auto attach works after the inode was once evicted from cache ========================================================== 19:38:04 (1746142684) 120+0 records in 120+0 records out 125829120 bytes (126 MB) copied, 0.20539 s, 613 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d20.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/122880 done Creating filesystem with 122880 1k blocks and 30720 inodes Filesystem UUID: 3d4a03cb-aec6-426a-a98f-83b04c6227ce Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/15 done Writing inode tables: 0/15 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/15 done /tmp/f20.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3d4a03cb-aec6-426a-a98f-83b04c6227ce (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d20.sanity-pcc/d20.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d20.sanity-pcc is idle now PASS 20 (5s) == sanity-pcc test 21a: PCC-RO storing as a plain HSM mirror component for plain layout ========================================================== 19:38:10 (1746142690) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864379 s, 607 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 415ce79d-5355-4b64-b3c8-ada53d237a74 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=415ce79d-5355-4b64-b3c8-ada53d237a74 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21a.sanity-pcc/d21a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Plain layout info before PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 267 0x10b 0x280000400 FLR layout info after PCC-RO attach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21a.sanity-pcc': /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC attach /mnt/lustre/f21a.sanity-pcc again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after RO-PCC detach '/mnt/lustre/f21a.sanity-pcc' again: /mnt/lustre/f21a.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x29 lmm_fid: [0x200003ab1:0x29:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10b:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21a.sanity-pcc is idle now PASS 21a (3s) == sanity-pcc test 21b: PCC-RO stroing as a plain HSM mirror component for FLR layouts ========================================================== 19:38:14 (1746142694) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0775532 s, 676 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 1de2aa7d-b013-4d71-9663-79aaae5c9ba7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=1de2aa7d-b013-4d71-9663-79aaae5c9ba7 (extents) (64bit) (large files) (huge files) FLR layout before PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after PCC-RO attach '/mnt/lustre/f21b.sanity-pcc' again: /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout info after PCC-RO detach '/mnt/lustre/f21b.sanity-pcc': /mnt/lustre/f21b.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10c:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x10c:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2a lmm_fid: [0x200003ab1:0x2a:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10d:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x10d:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21b.sanity-pcc is idle now PASS 21b (4s) == sanity-pcc test 21c: Verify HSM release works storing PCC-RO as HSM mirror component ========================================================== 19:38:18 (1746142698) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645996 s, 812 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: be9c69d6-14f8-49fa-943f-5a0a631fd276 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=be9c69d6-14f8-49fa-943f-5a0a631fd276 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21c.sanity-pcc/d21c.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Waiting 200s for 'SUCCEED' /mnt/lustre/f21c.sanity-pcc: (0x00000009) exists archived, archive_id:2 /mnt/lustre/f21c.sanity-pcc lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 0 lmm_stripe_offset: 0 lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' /mnt/lustre/f21c.sanity-pcc: (0x0000000d) released exists archived, archive_id:2 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21c.sanity-pcc is idle now PASS 21c (6s) == sanity-pcc test 21d: Write should invalidate PCC-RO caching ========================================================== 19:38:24 (1746142704) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720653 s, 728 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21d.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 30c93c3a-8ff8-42cb-a477-91fba7de941c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21d.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=30c93c3a-8ff8-42cb-a477-91fba7de941c (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21d.sanity-pcc/d21d.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21d.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 270 0x10e 0x280000400 PCC-RO attach '/mnt/lustre/f21d.sanity-pcc': /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Write invalidated PCC-RO cache: /mnt/lustre/f21d.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2d lmm_fid: [0x200003ab1:0x2d:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10e:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21d.sanity-pcc is idle now PASS 21d (4s) == sanity-pcc test 21e: Truncate should invalidate PCC-RO caching ========================================================== 19:38:28 (1746142708) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721369 s, 727 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21e.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 0ee20dab-edb3-4b03-97e5-a8b0e41fb8de Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21e.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0ee20dab-edb3-4b03-97e5-a8b0e41fb8de (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21e.sanity-pcc/d21e.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21e.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 271 0x10f 0x240000400 PCC-RO attach '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 8594128896 lhm_archive_ver: 9663677440 lhm_archive_uuid: '' Trucate invalidate PCC-RO file '/mnt/lustre/f21e.sanity-pcc': /mnt/lustre/f21e.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x2f lmm_fid: [0x200003ab1:0x2f:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21e.sanity-pcc is idle now PASS 21e (4s) == sanity-pcc test 21f: mmap write should invalidate PCC-RO caching ========================================================== 19:38:32 (1746142712) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788469 s, 665 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21f.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3c00ccb1-4543-41ea-966e-7094c7d7d040 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21f.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3c00ccb1-4543-41ea-966e-7094c7d7d040 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21f.sanity-pcc/d21f.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21f.sanity-pcc lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 271 0x10f 0x280000400 PCC-RO attach '/mnt/lustre/f21f.sanity-pcc': /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Mmap write invalidate PCC-RO caching: /mnt/lustre/f21f.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x30 lmm_fid: [0x200003ab1:0x30:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x10f:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 8594128896 lhm_archive_ver: 9663677440 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21f.sanity-pcc is idle now PASS 21f (5s) == sanity-pcc test 21g: PCC-RO for file under FLR write pending state ========================================================== 19:38:37 (1746142717) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900694 s, 582 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21g.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 85513720-44f1-4621-b68a-7a546e9d7406 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21g.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=85513720-44f1-4621-b68a-7a546e9d7406 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21g.sanity-pcc/d21g.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: wp lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp,pccro lcm_layout_gen: 4 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after Write invalidate '/mnt/lustre/f21g.sanity-pcc': /mnt/lustre/f21g.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: wp lcm_layout_gen: 5 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x110:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x110:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x32 lmm_fid: [0x200003ab1:0x32:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x111:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x111:0x0] } - lcme_id: 196612 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21g.sanity-pcc is idle now PASS 21g (4s) == sanity-pcc test 21h: Extend mirror once file was PCC-RO cached ========================================================== 19:38:41 (1746142721) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727877 s, 720 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21h.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 67d135f8-f215-428d-92e3-6fd72e2fcc66 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21h.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=67d135f8-f215-428d-92e3-6fd72e2fcc66 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21h.sanity-pcc/d21h.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' FLR layout before PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 288 lcm_flags: ro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 208 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc': /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 408 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 3 lcm_entry_count: 3 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 176 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 256 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 336 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' FLR layout after extend a mirror: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } file: /mnt/lustre/f21h.sanity-pcc, type: none FLR layout after PCC-RO attach '/mnt/lustre/f21h.sanity-pcc' again: /mnt/lustre/f21h.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 536 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 4 lcm_entry_count: 4 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 224 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 0 lmm_objects: - 0: { l_ost_idx: 0, l_fid: [0x240000400:0x112:0x0] } - 1: { l_ost_idx: 1, l_fid: [0x280000400:0x112:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 304 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x113:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x113:0x0] } - lcme_id: 196611 lcme_mirror_id: 3 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 384 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' - lcme_id: 262145 lcme_mirror_id: 4 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 456 lcme_size: 80 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x35 lmm_fid: [0x200003ab1:0x35:0x0] lmm_stripe_count: 2 lmm_stripe_size: 8388608 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x114:0x0] } - 1: { l_ost_idx: 0, l_fid: [0x240000400:0x114:0x0] } Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21h.sanity-pcc is idle now PASS 21h (4s) == sanity-pcc test 21i: HSM release increase layout gen, should invalidate PCC-RO cache ========================================================== 19:38:45 (1746142725) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835443 s, 628 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d21i.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5119226b-0c24-4637-a8e2-ab572fe3bad8 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f21i.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5119226b-0c24-4637-a8e2-ab572fe3bad8 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d21i.sanity-pcc/d21i.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' /mnt/lustre/f21i.sanity-pcc: (0x00000009) exists archived, archive_id:2 Layout after HSM release /mnt/lustre/f21i.sanity-pcc: /mnt/lustre/f21i.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 16 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC state /mnt/lustre/f21i.sanity-pcc: file: /mnt/lustre/f21i.sanity-pcc, type: none Layout after PCC-RO attach /mnt/lustre/f21i.sanity-pcc again: /mnt/lustre/f21i.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x37 lmm_fid: [0x200003ab1:0x37:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 1 obdidx objid objid group 1 278 0x116 0x280000400 PCC state: file: /mnt/lustre/f21i.sanity-pcc, type: readonly, PCC_file: /d21i.sanity-pcc/3a86/0x200003ab1:0x37:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d21i.sanity-pcc is idle now PASS 21i (6s) == sanity-pcc test 21j: PCC-RO for encrypted file ======== 19:38:51 (1746142731) SKIP: sanity-pcc test_21j client encryption not supported SKIP 21j (1s) == sanity-pcc test 22: Test RO-PCC attach for the HSM released file ========================================================== 19:38:53 (1746142733) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112502 s, 466 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d22.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 26aebdc4-69c1-440f-b450-d0501b5b07d9 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f22.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=26aebdc4-69c1-440f-b450-d0501b5b07d9 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d22.sanity-pcc/d22.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 3 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 2 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x117:0x0] } - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after HSM release /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: released lmm_layout_gen: 2 lmm_stripe_offset: 16 - lcme_id: 131075 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Layout after PCC-RO attach /mnt/lustre/f22.sanity-pcc again: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 0 obdidx objid objid group 0 279 0x117 0x240000400 Layout after PCC-RO detach /mnt/lustre/f22.sanity-pcc: /mnt/lustre/f22.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x3c lmm_fid: [0x200003ab1:0x3c:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 5 lmm_stripe_offset: 0 obdidx objid objid group 0 279 0x117 0x240000400 Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d22.sanity-pcc is idle now PASS 22 (11s) == sanity-pcc test 23: Test write/truncate/mmap-write invalidating RO-PCC caching ========================================================== 19:39:04 (1746142744) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771293 s, 680 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d23.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 374ecca4-6aa3-4c32-a453-95f701c80838 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f23.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=374ecca4-6aa3-4c32-a453-95f701c80838 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d23.sanity-pcc/d23.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Write should invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 4 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init,stale lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,dirty,archived,0xb lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 5 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 8594128896 lhm_archive_ver: 10737419264 lhm_archive_uuid: '' Truncate invalidate the RO-PCC cache: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Finish trucate operation Mmap write invalidates RO-PCC caching /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 6 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' PCC-RO attach '/mnt/lustre/f23.sanity-pcc' again: /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp,pccro lcm_layout_gen: 7 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Mmap write /mnt/lustre/f23.sanity-pcc via multiop /mnt/lustre/f23.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: wp lcm_layout_gen: 8 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x200003ab1 lmm_object_id: 0x44 lmm_fid: [0x200003ab1:0x44:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x119:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,0x9 lhm_archive_id: 343597384136 lhm_archive_ver: 0 lhm_archive_uuid: '' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d23.sanity-pcc is idle now PASS 23 (6s) == sanity-pcc test 24a: Test RO-PCC with non-root user === 19:39:10 (1746142750) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0842109 s, 623 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: e7a79498-39d6-4a82-b17f-c1ec9db05072 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e7a79498-39d6-4a82-b17f-c1ec9db05072 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: a0 autocache: projid={100} running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00928236 s, 110 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00627124 s, 163 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [-k] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] touch: cannot touch '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0] [bs=1024] [count=1] dd: failed to open '/mnt/pcc.d24a.sanity-pcc/d24a.sanity-pcc/3af6/0x200003ab1:0x47:0x0': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [-i] [2] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24a.sanity-pcc/f24a.sanity-pcc] Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24a.sanity-pcc is idle now PASS 24a (5s) == sanity-pcc test 24b: General permission test for RO-PCC ========================================================== 19:39:15 (1746142755) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673518 s, 778 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d24b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: a3b96cfb-81f5-4280-8bf4-00355748eaff Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f24b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a3b96cfb-81f5-4280-8bf4-00355748eaff (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d24b.sanity-pcc/d24b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0053147 s, 193 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/dev/zero] [of=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] dd: failed to open '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc': Permission denied running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00186144 s, 550 kB/s running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] detach: cannot detach '/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc' from PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [dd] [if=/mnt/lustre/d24b.sanity-pcc/f24b.sanity-pcc] [of=/dev/null] [bs=1024] [count=1] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00280896 s, 365 kB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d24b.sanity-pcc is idle now PASS 24b (5s) == sanity-pcc test 25: Tolerate fake read failure for RO-PCC ========================================================== 19:39:20 (1746142760) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762523 s, 688 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d25.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 310a6ff4-0632-431d-bfed-6b5668777a09 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f25.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=310a6ff4-0632-431d-bfed-6b5668777a09 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d25.sanity-pcc/d25.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' fail_loc=0x1411 fail_loc=0 fail_loc=0x1411 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d25.sanity-pcc is idle now PASS 25 (6s) == sanity-pcc test 26: Repeat the attach/detach when the file has multiple openers ========================================================== 19:39:26 (1746142766) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710395 s, 738 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d26.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 34ebcaa2-1fad-483a-8a13-187c87512a41 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f26.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=34ebcaa2-1fad-483a-8a13-187c87512a41 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d26.sanity-pcc/d26.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2521 client oleg413-client.virtnet multiop_bg started multiop_pid=2532 Stopping multiop_pid=2532 (kill 2532 on oleg413-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2636 client oleg413-client.virtnet multiop_bg started multiop_pid=2646 Stopping multiop_pid=2646 (kill 2646 on oleg413-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2751 client oleg413-client.virtnet multiop_bg started multiop_pid=2761 Stopping multiop_pid=2761 (kill 2761 on oleg413-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.2866 client oleg413-client.virtnet multiop_bg started multiop_pid=2876 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=2876 (kill 2876 on oleg413-client.virtnet) multiop /mnt/lustre/f26.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.3016 client oleg413-client.virtnet multiop_bg started multiop_pid=3026 Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=3026 (kill 3026 on oleg413-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d26.sanity-pcc is idle now PASS 26 (28s) == sanity-pcc test 27: Auto attach at open when the file has multiple openers ========================================================== 19:39:54 (1746142794) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107558 s, 487 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d27.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 6a36a6b9-1b40-493d-b97d-a06f3c21b4dd Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f27.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6a36a6b9-1b40-493d-b97d-a06f3c21b4dd (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d27.sanity-pcc/d27.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.3933 client oleg413-client.virtnet multiop_bg started multiop_pid=3944 Waiting 200s for 'SUCCEED' Stopping multiop_pid=3944 (kill 3944 on oleg413-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4077 client oleg413-client.virtnet multiop_bg started multiop_pid=4087 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear Waiting 200s for 'SUCCEED' Updated after 2s: want 'SUCCEED' got 'SUCCEED' Stopping multiop_pid=4087 (kill 4087 on oleg413-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4221 client oleg413-client.virtnet multiop_bg started multiop_pid=4231 Stopping multiop_pid=4231 (kill 4231 on oleg413-client.virtnet) multiop /mnt/lustre/f27.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.4340 client oleg413-client.virtnet multiop_bg started multiop_pid=4350 ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af518000.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff880130f34000.lru_size=clear Stopping multiop_pid=4350 (kill 4350 on oleg413-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d27.sanity-pcc is idle now PASS 27 (25s) == sanity-pcc test 28: RW-PCC attach should fail when the file has cluster-wide openers ========================================================== 19:40:19 (1746142819) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146309 s, 358 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d28.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: c9898737-5612-4fd6-b95e-4650fb6075f1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f28.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=c9898737-5612-4fd6-b95e-4650fb6075f1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d28.sanity-pcc/d28.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.5209 client oleg413-client.virtnet multiop_bg started multiop_pid=5220 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Stopping multiop_pid=5220 (kill 5220 on oleg413-client.virtnet) multiop /mnt/lustre2/f28.sanity-pcc vO_c TMPPIPE=/tmp/multiop_open_wait_pipe.7025 lfs pcc pcc: cannot get WRITE lease, ext 0 for [0x200003ab1:0x5f:0x0]: Device or resource busy (16) lfs pcc pcc: cannot get lease: Device or resource busy (16) attach: cannot attach '/mnt/lustre/f28.sanity-pcc' to PCC with attach ID '2': Device or resource busy Cleanup PCC backend on /mnt/lustre /mnt/pcc.d28.sanity-pcc is idle now PASS 28 (10s) == sanity-pcc test 29a: Auto readonly caching on RO-PCC backend for O_RDONLY open ========================================================== 19:40:29 (1746142829) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800aafda000.idle_timeout=debug osc.lustre-OST0000-osc-ffff880130f33800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aafda000.idle_timeout=debug osc.lustre-OST0001-osc-ffff880130f33800.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716773 s, 731 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 823199f8-39a1-44cf-8cf3-e6273043dbb4 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=823199f8-39a1-44cf-8cf3-e6273043dbb4 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d29a.sanity-pcc/d29a.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: ae autocache: projid={100} 100 P /mnt/lustre/d29a.sanity-pcc/f29a.sanity-pcc Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29a.sanity-pcc is idle now PASS 29a (50s) == sanity-pcc test 29b: Auto PCC-RO attach in atomic_open ========================================================== 19:41:19 (1746142879) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0722929 s, 725 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d29b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b061c4dc-f0b5-4cab-a342-ee408140cc2b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f29b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b061c4dc-f0b5-4cab-a342-ee408140cc2b (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d29b.sanity-pcc/d29b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.dat} 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00663754 s, 617 kB/s 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00279253 s, 1.5 MB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00392072 s, 1.0 MB/s vm.drop_caches = 3 1+0 records in 1+0 records out 4096 bytes (4.1 kB) copied, 0.00585641 s, 699 kB/s file: /mnt/lustre/myfile.dat, type: readonly, PCC_file: /d29b.sanity-pcc/4282/0x200004281:0x3:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d29b.sanity-pcc is idle now PASS 29b (5s) == sanity-pcc test 30: Test lctl pcc del command ========= 19:41:25 (1746142885) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0901556 s, 582 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d30.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 46c2332e-43f7-4956-9128-46d245631561 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f30.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=46c2332e-43f7-4956-9128-46d245631561 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 3e autocache: projid={100} check file 0:'428b' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'4287' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 0:'4289' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc' check file 1:'0x200004281:0xa:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/428b' Handle the file: 0x200004281:0xa:0x0 Detach and remove the PCC cached file: 0x200004281:0xa:0x0 check file 1:'0x200004281:0x6:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4287' Handle the file: 0x200004281:0x6:0x0 Detach and remove the PCC cached file: 0x200004281:0x6:0x0 check file 1:'0x200004281:0x8:0x0' under directory '/mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289' Handle the file: 0x200004281:0x8:0x0 Remove non-cached file: /mnt/pcc.d30.sanity-pcc/d30.sanity-pcc/4289/0x200004281:0x8:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d30.sanity-pcc is idle now PASS 30 (5s) == sanity-pcc test 31: Test lctl pcc del command with --keep option ========================================================== 19:41:29 (1746142889) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755535 s, 694 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d31.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 3ddf913c-578c-4246-add3-91b37a4dfc37 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f31.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=3ddf913c-578c-4246-add3-91b37a4dfc37 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d31.sanity-pcc/d31.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d31.sanity-pcc/d31.sanity-pcc hsmtool: posix_v2 rwid: 2 roid: 2 flags: 30 autocache: projid={100} Cleanup PCC backend on /mnt/lustre /mnt/pcc.d31.sanity-pcc is idle now PASS 31 (4s) == sanity-pcc test 32: Test for RO-PCC when PCC copy is deleted ========================================================== 19:41:33 (1746142893) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0817757 s, 641 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d32.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 71379785-68dd-4f88-90d5-88f192a13e39 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f32.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=71379785-68dd-4f88-90d5-88f192a13e39 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d32.sanity-pcc/d32.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' multiop /mnt/lustre/f32.sanity-pcc vo_rc TMPPIPE=/tmp/multiop_open_wait_pipe.11486 client oleg413-client.virtnet multiop_bg started multiop_pid=11496 Stopping multiop_pid=11496 (kill 11496 on oleg413-client.virtnet) Cleanup PCC backend on /mnt/lustre /mnt/pcc.d32.sanity-pcc is idle now PASS 32 (11s) == sanity-pcc test 33: Cache rule with comparator (>, =, <) for file size ========================================================== 19:41:44 (1746142904) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0897344 s, 584 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d33.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: cfe6be2c-1ebf-44c4-be6f-559f194f98d1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f33.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=cfe6be2c-1ebf-44c4-be6f-559f194f98d1 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d33.sanity-pcc/d33.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: fname={*.doc}&size<{1M} file: /mnt/lustre/myfile.doc, type: none file: /mnt/lustre/myfile.doc, type: none Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size<{5M}&size>{3M} Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d33.sanity-pcc/d33.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: fname={*.doc}&size={5M 3M} Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d33.sanity-pcc is idle now PASS 33 (5s) == sanity-pcc test 34: Cache rule with comparator (>, <) for Project ID range ========================================================== 19:41:49 (1746142909) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions libkmod: kmod_module_get_holders: could not open '/sys/module/pcc_cpufreq/holders': No such file or directory loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions oleg413-server: libkmod: kmod_module_get_holders: could not open '/sys/module/acpi_cpufreq/holders': No such file or directory Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800b63eb000.idle_timeout=debug osc.lustre-OST0000-osc-ffff88013767e000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800b63eb000.idle_timeout=debug osc.lustre-OST0001-osc-ffff88013767e000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569134 s, 921 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d34.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 092f845a-6e90-4155-a4cc-cf595ac065c7 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f34.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=092f845a-6e90-4155-a4cc-cf595ac065c7 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d34.sanity-pcc/d34.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid>{100} 99 - /mnt/lustre/f34.sanity-pcc 101 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{100} 102 - /mnt/lustre/f34.sanity-pcc 99 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre pcc: - pccpath: /mnt/pcc.d34.sanity-pcc/d34.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 5 flags: ae autocache: projid<{120}&projid>{110} 105 - /mnt/lustre/f34.sanity-pcc 121 - /mnt/lustre/f34.sanity-pcc 115 - /mnt/lustre/f34.sanity-pcc Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre Cleanup PCC backend on /mnt/lustre /mnt/pcc.d34.sanity-pcc is idle now PASS 34 (52s) == sanity-pcc test 35: mmap fault test =================== 19:42:41 (1746142961) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741714 s, 707 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d35.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: cc45c0d3-065e-4119-acaa-774f1282fc14 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f35.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=cc45c0d3-065e-4119-acaa-774f1282fc14 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d35.sanity-pcc/d35.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d35.sanity-pcc is idle now PASS 35 (3s) == sanity-pcc test 36a: Stale RW-PCC copy should be deleted after remove the PCC backend ========================================================== 19:42:44 (1746142964) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732439 s, 716 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 001dcd04-d361-4ae8-807b-217ef6c855a5 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=001dcd04-d361-4ae8-807b-217ef6c855a5 (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a52' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc' check file 1:'0x200004a51:0x3:0x0' under directory '/mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52' Handle the file: 0x200004a51:0x3:0x0 Remove non-cached file: /mnt/pcc.d36a.sanity-pcc/d36a.sanity-pcc/4a52/0x200004a51:0x3:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36a.sanity-pcc is idle now PASS 36a (4s) == sanity-pcc test 36b: Stale RO-PCC copy should be deleted after remove the PCC backend ========================================================== 19:42:48 (1746142968) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648573 s, 808 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d36b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5acd7bc0-3a3b-4f6b-a9ef-7ce248bc76da Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f36b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5acd7bc0-3a3b-4f6b-a9ef-7ce248bc76da (extents) (64bit) (large files) (huge files) Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' check file 0:'4a54' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc' check file 1:'0x200004a51:0x5:0x0' under directory '/mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54' Handle the file: 0x200004a51:0x5:0x0 Remove non-cached file: /mnt/pcc.d36b.sanity-pcc/d36b.sanity-pcc/4a54/0x200004a51:0x5:0x0 flags: 1 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d36b.sanity-pcc is idle now PASS 36b (4s) == sanity-pcc test 37: Multiple readers on a shared file with PCC-RO mode ========================================================== 19:42:52 (1746142972) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721903 s, 726 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d37.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: b8d06c6d-4936-4190-9c12-dea251a0149b Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=b8d06c6d-4936-4190-9c12-dea251a0149b (extents) (64bit) (large files) (huge files) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682443 s, 768 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d37.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f37.sanity-pcc on /mnt/pcc.d37.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 40839f1c-a5af-4b54-bf45-bf58e129ccd1 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f37.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=40839f1c-a5af-4b54-bf45-bf58e129ccd1 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d37.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} pcc: - pccpath: /mnt/pcc.d37.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: a0 autocache: projid={2} file: /mnt/lustre/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 file: /mnt/lustre2/d37.sanity-pcc/f37.sanity-pcc, type: readonly, PCC_file: /4a56/0x200004a51:0x7:0x0, open_count: 0, flags: 0 /mnt/pcc.d37.sanity-pcc.2 is idle now /mnt/pcc.d37.sanity-pcc is idle now PASS 37 (6s) == sanity-pcc test 38: Verify LFS pcc state does not trigger prefetch for auto PCC-RO ========================================================== 19:42:58 (1746142978) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a5688000.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800aafde000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a5688000.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800aafde000.idle_timeout=debug disable quota as required 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0684067 s, 766 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d38.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 5415f08d-95c7-4499-b4ea-70175772b9dd Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f38.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5415f08d-95c7-4499-b4ea-70175772b9dd (extents) (64bit) (large files) (huge files) file: /mnt/lustre/d38.sanity-pcc/f38.sanity-pcc, type: none QQQQQ Cleanup PCC backend on /mnt/lustre /mnt/pcc.d38.sanity-pcc is idle now PASS 38 (63s) == sanity-pcc test 39: Test Project quota on loop PCC device ========================================================== 19:44:01 (1746143041) SKIP: sanity-pcc test_39 Not support project quota on local filesystem SKIP 39 (1s) == sanity-pcc test 40: Test async open attach in the background for PCC-RO file ========================================================== 19:44:02 (1746143042) using SAVE_PROJECT_SUPPORTED=0 Stopping clients: oleg413-client.virtnet /mnt/lustre (opts:) Stopping client oleg413-client.virtnet /mnt/lustre opts: Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: Stopping /mnt/lustre-mds1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost1 (opts:-f) on oleg413-server Stopping /mnt/lustre-ost2 (opts:-f) on oleg413-server sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) Checking servers environments Checking clients oleg413-client.virtnet environments Loading modules from /home/green/git/lustre-release/lustre detected 4 online CPUs by sysfs MODOPTS_LIBCFS= Force libcfs to create 2 CPU partitions loading modules on: 'oleg413-server' oleg413-server: oleg413-server.virtnet: executing load_modules_local oleg413-server: Loading modules from /home/green/git/lustre-release/lustre oleg413-server: detected 4 online CPUs by sysfs oleg413-server: MODOPTS_LIBCFS= oleg413-server: Force libcfs to create 2 CPU partitions Setup mgs, mdt, osts Starting mds1: -o localrecov lustre-mdt1/mdt1 /mnt/lustre-mds1 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-MDT0000 Starting ost1: -o localrecov lustre-ost1/ost1 /mnt/lustre-ost1 seq.cli-lustre-OST0000-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0000 Starting ost2: -o localrecov lustre-ost2/ost2 /mnt/lustre-ost2 seq.cli-lustre-OST0001-super.width=65536 oleg413-server: oleg413-server.virtnet: executing set_default_debug vfstrace rpctrace dlmtrace neterror ha config ioctl super lfsck all pdsh@oleg413-client: oleg413-server: ssh exited with exit code 1 Started lustre-OST0001 Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Starting client: oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Starting client oleg413-client.virtnet: -o user_xattr,flock 192.168.204.113@tcp:/lustre /mnt/lustre2 Started clients oleg413-client.virtnet: 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) Using TIMEOUT=20 osc.lustre-OST0000-osc-ffff8800a8819800.idle_timeout=debug osc.lustre-OST0000-osc-ffff8800af4f4800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800a8819800.idle_timeout=debug osc.lustre-OST0001-osc-ffff8800af4f4800.idle_timeout=debug disable quota as required 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.243037 s, 863 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d40.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: 6f263e00-ad32-460d-8443-da7b75548902 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f40.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6f263e00-ad32-460d-8443-da7b75548902 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d40.sanity-pcc/d40.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.378674 s, 138 MB/s 100 - /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none ldlm.namespaces.lustre-OST0000-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800af4f4800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800af4f4800.lru_size=clear osc.lustre-OST0000-osc-ffff8800a8819800.stats=clear osc.lustre-OST0000-osc-ffff8800af4f4800.stats=clear osc.lustre-OST0001-osc-ffff8800a8819800.stats=clear osc.lustre-OST0001-osc-ffff8800af4f4800.stats=clear fail_val=1 fail_loc=0x214 llite.lustre-ffff8800a8819800.pcc_async_threshold=5MB llite.lustre-ffff8800af4f4800.pcc_async_threshold=5MB Test open attach with pcc_async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 2.04114 s, 514 kB/s Before: 0 After: 3 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none, flags: attaching /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 8s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800a8819800.pcc_async_threshold=1G llite.lustre-ffff8800af4f4800.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800af4f4800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800af4f4800.lru_size=clear osc.lustre-OST0000-osc-ffff8800a8819800.stats=clear osc.lustre-OST0000-osc-ffff8800af4f4800.stats=clear osc.lustre-OST0001-osc-ffff8800a8819800.stats=clear osc.lustre-OST0001-osc-ffff8800af4f4800.stats=clear Test open attach with async_threshold=1G file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: readonly, PCC_file: /d40.sanity-pcc/59f3/0x2000059f1:0x2:0x0, open_count: 0, flags: 0 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00883949 s, 119 MB/s Before: 50 After: 50 Time1: 2 Time2: 10 file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800a8819800.pcc_async_threshold=5MB llite.lustre-ffff8800af4f4800.pcc_async_threshold=5MB ldlm.namespaces.lustre-OST0000-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800af4f4800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800af4f4800.lru_size=clear Read 1MB data with async_threshold=5MB 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 1.04098 s, 1.0 MB/s /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc | grep -E -c 'type: readonly' Waiting 50s for '1' Updated after 9s: want '1' got '1' file: /mnt/lustre/d40.sanity-pcc/f40.sanity-pcc, type: none llite.lustre-ffff8800a8819800.pcc_async_threshold=1G llite.lustre-ffff8800af4f4800.pcc_async_threshold=1G ldlm.namespaces.lustre-OST0000-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0000-osc-ffff8800af4f4800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-OST0001-osc-ffff8800af4f4800.lru_size=clear Read 1MB data with async_threshold=1G 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.00475083 s, 221 MB/s Time1: 1 Time2: 9 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d40.sanity-pcc is idle now PASS 40 (108s) == sanity-pcc test 41: Test mtime rule for PCC-RO open attach with O_RDONLY mode ========================================================== 19:45:50 (1746143150) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791718 s, 662 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d41.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 107f860d-c726-4212-9ad1-fa1454f8f297 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f41.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=107f860d-c726-4212-9ad1-fa1454f8f297 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d41.sanity-pcc/d41.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: mtime>{1m} pcc_ro_data pcc_ro_data Cleanup PCC backend on /mnt/lustre /mnt/pcc.d41.sanity-pcc is idle now PASS 41 (3s) == sanity-pcc test 42: PCC attach without attach ID specified ========================================================== 19:45:53 (1746143153) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0776103 s, 811 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d42.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 06825335-bcdc-4e0d-9854-d9a130680708 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f42.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=06825335-bcdc-4e0d-9854-d9a130680708 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d42.sanity-pcc/d42.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={100} file: /mnt/lustre/f42.sanity-pcc, type: readonly, PCC_file: /d42.sanity-pcc/59f5/0x2000059f1:0x4:0x0, open_count: 0, flags: 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d42.sanity-pcc is idle now PASS 42 (3s) == sanity-pcc test 43: Auto attach at open() should add capacity owner check ========================================================== 19:45:56 (1746143156) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0898517 s, 700 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d43.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 6d20eb7a-bdbf-4b27-a0e9-156c9c890500 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f43.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=6d20eb7a-bdbf-4b27-a0e9-156c9c890500 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d43.sanity-pcc/d43.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: size<{100M} file: /mnt/lustre/f43.sanity-pcc, type: none running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f43.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f43.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/f43.sanity-pcc] file: /mnt/lustre/f43.sanity-pcc, type: none Cleanup PCC backend on /mnt/lustre /mnt/pcc.d43.sanity-pcc is idle now PASS 43 (4s) == sanity-pcc test 44: Verify valid auto attach without re-fetching the whole files ========================================================== 19:46:00 (1746143160) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0945346 s, 666 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d44.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 8c56e206-da69-4bd8-b490-6ec1e480c198 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f44.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=8c56e206-da69-4bd8-b490-6ec1e480c198 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d44.sanity-pcc/d44.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8819800.pcc_async_threshold=0 llite.lustre-ffff8800af4f4800.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.475385 s, 110 MB/s /mnt/lustre/f44.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 420 0x1a4 0x280000400 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.458267 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.508479 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.532695 s, 98.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.463443 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.536471 s, 97.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.537059 s, 97.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.522283 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.510085 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.549968 s, 95.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.472057 s, 111 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.557803 s, 94.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.482649 s, 109 MB/s 52428800 bytes (52 MB) copied, 0.537022 s, 97.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.585621 s, 89.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515707 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.573641 s, 91.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224234 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250632 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228936 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172839 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226636 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311646 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236501 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169188 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365708 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223992 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268852 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232225 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440216 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290963 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044944 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225171 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034192 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227651 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221097 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372392 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319378 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408205 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298736 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353026 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465846 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596196 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342685 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215152 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444464 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270165 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351671 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360348 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269263 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273122 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331716 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191296 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174145 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212519 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317396 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210804 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281287 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425841 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432993 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601883 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190103 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323673 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242698 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389015 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229391 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349961 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045053 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326114 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310739 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265793 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427284 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376238 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049785 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0502579 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0410167 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045998 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330384 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420934 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455644 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294375 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274562 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282214 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183886 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487857 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510481 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395876 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269296 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545671 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601571 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0937133 s, 559 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324028 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574038 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0709016 s, 739 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503474 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300512 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317122 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275882 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546584 s, 959 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595803 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448111 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616428 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292267 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02074 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238321 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240842 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0171153 s, 3.1 GB/s , 0.0171174 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026313 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246263 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171342 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502996 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276088 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029555 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511892 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369249 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322285 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277963 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400812 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169445 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530784 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175409 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060023 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265877 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065501 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343361 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467414 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522968 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044033 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249637 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209731 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176777 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485356 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554888 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616864 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065631 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259373 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286436 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260687 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333553 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034274 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248438 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167493 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196753 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207255 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426071 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374838 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063521 s, 825 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480943 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181581 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369352 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424821 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246212 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494037 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255651 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576795 s, 909 MB/s 52428800 bytes (52 MB) copied, 0.0251919 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742454 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171595 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035048 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806924 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522085 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193953 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235074 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262348 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279785 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250556 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392358 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233607 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251448 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360229 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194628 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216792 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332996 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183529 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03557 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236871 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257185 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178996 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020509 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388516 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213597 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205486 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307207 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308573 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412207 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430883 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039202 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401706 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314704 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319414 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532433 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258079 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295715 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659903 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349278 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298038 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025496 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030685 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472921 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238501 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244277 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255316 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484802 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029119 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221955 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185827 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331428 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463175 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731436 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020525 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308537 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197853 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.042338 s, 1.2 GB/s , 0.0249291 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355767 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02018 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188076 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210659 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259685 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338919 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272332 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280292 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037329 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211851 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259733 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210768 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020417 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197791 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231991 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280208 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572206 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535885 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491357 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292615 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236275 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308171 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026178 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339388 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614687 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0768219 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526514 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438155 s, 1.2 GB/s , 0.0600997 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372116 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325311 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207108 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320814 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178122 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803168 s, 653 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172168 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708379 s, 740 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281431 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342509 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339247 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467988 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040237 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350703 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213443 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211305 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182082 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274123 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229157 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196923 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403547 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445084 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535942 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240003 s, 2.2 GB/s , 0.0586646 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262471 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338552 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274936 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319685 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283712 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030304 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029394 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035409 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360242 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369756 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504549 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0199639 s, 2.6 GB/s , 0.0163591 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366975 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425392 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022928 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263947 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289162 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209206 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210139 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381017 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329363 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353796 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190885 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261968 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213761 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239401 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251258 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257251 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.079244 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339354 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382207 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489503 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563806 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240424 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248249 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310591 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241244 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407871 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246102 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02243 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232337 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205955 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239214 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237169 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230658 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025352 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213441 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226107 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049391 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216808 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0487027 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480493 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338724 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220352 s, 2.4 GB/s , 0.0263186 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273775 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218023 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210001 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318804 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259446 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254622 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175281 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264434 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022965 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01747 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019219 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214707 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258187 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194936 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0244196 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171034 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167954 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188623 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334676 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305422 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259762 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019525 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353053 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225045 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200389 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248495 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244093 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184798 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191322 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170503 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251997 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218363 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267876 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196531 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220469 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189408 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197807 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021573 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196346 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207687 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236697 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478398 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052167 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454971 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393806 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258278 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299929 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294222 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327924 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350486 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306951 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304891 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253901 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0165945 s, 3.2 GB/s , 0.0240927 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289485 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272229 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262716 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257039 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179972 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317975 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205964 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018964 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267435 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274647 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019582 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026535 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287958 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306874 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220527 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281362 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193549 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028117 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038396 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265847 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019811 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189202 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289496 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276334 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188289 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205249 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234215 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222067 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273905 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251852 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267235 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0267833 s, 2.0 GB/s , 0.0220394 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271282 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212103 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017652 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284556 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403898 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017504 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248838 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0222175 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266948 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169507 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279298 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245708 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296222 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350893 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350734 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171648 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168247 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198465 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363409 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233189 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268102 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257663 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178757 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173123 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242585 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256473 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198157 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166952 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233392 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0284465 s, 1.8 GB/s , 0.0295362 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184539 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316375 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188697 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020799 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209245 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244822 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033032 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317317 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489977 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265519 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290917 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180542 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229468 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463666 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291575 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271911 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261472 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216889 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328454 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303786 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256453 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215637 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182783 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240685 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334016 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172233 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054239 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404266 s, 1.3 GB/s , 0.0319146 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460072 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414781 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054434 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169503 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344381 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221226 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244582 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342134 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173297 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317692 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258741 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310193 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01955 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025543 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487718 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472057 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501869 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030005 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330476 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328405 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236743 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319087 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271131 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223722 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204362 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257925 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272485 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452908 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03346 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057187 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389095 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374601 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446091 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273354 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297964 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258814 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213385 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339696 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020744 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275493 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225194 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241558 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0443914 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209049 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391463 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031843 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228277 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209234 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235315 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162626 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02432 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022614 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316579 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237033 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274218 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299785 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224336 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277579 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423107 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309634 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288542 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390898 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252487 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324414 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199158 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191126 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262106 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035615 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319166 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0448801 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226091 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018442 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286742 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0250867 s, 2.1 GB/s , 0.027245 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281238 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221479 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254957 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194695 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0261893 s, 2.0 GB/s , 0.0311587 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282813 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171399 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231038 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230779 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019917 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024245 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227422 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178489 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0371394 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272429 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197316 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018942 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194773 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243643 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190179 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.031956 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245964 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424297 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032099 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267307 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235461 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276383 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264971 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221106 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194046 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273301 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033573 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340075 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204356 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234515 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022222 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0177349 s, 3.0 GB/s , 0.017732 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201109 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202781 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220204 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204993 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255686 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215888 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223959 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189105 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214827 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221295 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233325 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207597 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260024 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221316 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186969 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352832 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046766 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205811 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206853 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304634 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351702 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202612 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295742 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01981 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257611 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215972 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247159 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212236 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188701 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182857 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185337 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280901 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207181 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288176 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418421 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274574 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235479 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156596 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185159 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603004 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281568 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0568744 s, 922 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177703 s, 3.0 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0336025 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0406358 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198024 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269479 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172134 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0295573 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.022536 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296379 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286909 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042568 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221444 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0256337 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018229 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168262 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297746 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273831 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186557 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270394 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213967 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285952 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201354 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241528 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306745 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347191 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231414 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242724 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196022 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217519 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207643 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182542 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198594 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225228 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224665 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216909 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286297 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213527 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295371 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167663 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350316 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393213 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450824 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389142 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287213 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343449 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266216 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251753 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312173 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394563 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269368 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306227 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167404 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367437 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340615 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354841 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019219 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265711 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230329 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199045 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175838 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291753 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294993 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247588 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191879 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315921 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369073 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224279 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268488 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406203 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372162 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322825 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312304 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269998 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177348 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214224 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166997 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017602 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198929 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219103 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184306 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176967 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017435 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155949 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252161 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181776 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276541 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211457 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297329 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017397 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268724 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024211 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200578 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167494 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019357 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174739 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210152 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258819 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0187226 s, 2.8 GB/s , 0.0187605 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208737 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277705 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296992 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235139 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187948 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219418 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398693 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329954 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203486 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319475 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207147 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220499 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250795 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201547 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219859 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0165493 s, 3.2 GB/s , 0.0165475 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200872 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203588 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243647 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239213 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301321 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206082 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019985 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248613 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161487 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179723 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160315 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215788 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233167 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154907 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227253 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171437 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262245 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029975 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318565 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199957 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0237574 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247032 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376367 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270807 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222407 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442474 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020619 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406324 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286562 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257375 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232423 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223145 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189678 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022304 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228382 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208425 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236823 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205221 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017925 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239192 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342791 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270286 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250393 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321642 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181781 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213085 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170577 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234806 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175988 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163841 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160918 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223114 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236746 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236834 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173987 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216717 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188584 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278458 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293351 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252844 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213882 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230552 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257489 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372645 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437984 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407927 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353022 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382677 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319506 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303458 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177962 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265338 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163746 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269554 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165182 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234308 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179494 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264956 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281707 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300248 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022475 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217749 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232315 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0358831 s, 1.5 GB/s , 0.018621 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023602 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183038 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225015 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232955 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221509 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172548 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207448 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197122 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232955 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204688 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176259 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243746 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155553 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173221 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242484 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185141 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0158116 s, 3.3 GB/s , 0.0168086 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183147 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203557 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017715 s, 3.0 GB/s 52428800 bytes (52 MB) copied, 0.0159115 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170569 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027435 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154322 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194371 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233055 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0252673 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022168 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211388 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024984 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245513 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264623 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161274 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457739 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212965 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281556 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227412 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017161 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202424 s, 2.6 GB/s , 0.0183797 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161853 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292128 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219997 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159447 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178367 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171284 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272278 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016401 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236304 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229977 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233092 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203484 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021128 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183308 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215678 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175706 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170263 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285415 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282798 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177814 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188367 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228052 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244573 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213234 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256231 s, 2.0 GB/s , 0.0284039 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218254 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237985 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305702 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339494 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421619 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265575 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222318 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260328 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345124 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197469 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021398 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226843 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196123 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173737 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228872 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166129 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208559 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232891 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273124 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039282 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382078 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302429 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257137 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343964 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248614 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032355 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0304517 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021555 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220697 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409885 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0488504 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0301336 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0346097 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341873 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213441 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204606 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203488 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223566 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241672 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460783 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210005 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266711 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268102 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311938 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0310173 s, 1.7 GB/s , 0.0189529 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023933 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188004 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265527 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01634 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189389 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178854 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338263 s, 1.5 GB/s , 0.0355383 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480732 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030049 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018093 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390361 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475163 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155859 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205088 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508491 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247376 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192578 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184496 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232351 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259339 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221476 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190236 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200719 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294804 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054589 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336697 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325628 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264382 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0999656 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03261 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263968 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558798 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236655 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296247 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368729 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264501 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355222 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286791 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361929 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170797 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352551 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327227 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419471 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405554 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261002 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278061 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248295 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377253 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021958 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254427 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270517 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355312 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245256 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653878 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339378 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192197 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191689 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0167123 s, 3.1 GB/s , 0.0266292 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226102 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243556 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182006 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217841 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242654 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250818 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197131 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210312 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230741 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411816 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284176 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348119 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223441 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212766 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243502 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694708 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454558 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0831587 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453147 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379531 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471058 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0345851 s, 1.5 GB/s , 0.0218353 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428528 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200375 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219064 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204673 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191701 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192504 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038843 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379537 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625282 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673757 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0730097 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236664 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292122 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308905 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324971 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248654 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200482 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204089 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216346 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167455 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229431 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170803 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153336 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197237 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363447 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179521 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214073 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251105 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297191 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359647 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467155 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374834 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340679 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287055 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241701 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228675 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254802 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195225 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202449 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237008 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325291 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298472 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208527 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220077 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318794 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203846 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205746 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237293 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272729 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252397 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184987 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172727 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253177 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179158 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267593 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204732 s, 2.6 GB/s , 0.0273814 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286664 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028616 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179661 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163694 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310176 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385282 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406522 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361871 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558708 s, 938 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0635363 s, 825 MB/s , 0.0460923 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273091 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274633 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688469 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178274 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196357 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211368 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282901 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242882 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182009 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274049 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193743 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254009 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404761 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474772 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169939 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0533086 s, 983 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332402 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217004 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194187 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0159669 s, 3.3 GB/s , 0.0190422 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153969 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206412 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272511 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332309 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286981 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0247859 s, 2.1 GB/s , 0.0232291 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0446543 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504963 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212464 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185291 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409042 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016372 s, 3.2 GB/s 52428800 bytes (52 MB) copied, 0.0164642 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174909 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021018 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292113 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229026 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157941 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225299 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198513 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179597 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252444 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253973 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035652 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208717 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214603 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180168 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175037 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151238 s, 3.5 GB/s 52428800 bytes (52 MB) copied, 0.0159747 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213311 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251639 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220944 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282285 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268609 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365395 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315388 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183153 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257302 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260073 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386158 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164337 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210875 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201979 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186678 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262582 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264586 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224668 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191103 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330369 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228241 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159768 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213483 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0239579 s, 2.2 GB/s , 0.0202232 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179919 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193709 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217187 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211659 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026439 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175269 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182137 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196039 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211854 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165309 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265885 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173279 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245053 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225999 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486207 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0365258 s, 1.4 GB/s , 0.0226513 s, 2.3 GB/s , 0.0369397 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245316 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261461 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474207 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301228 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225152 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188126 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186536 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186995 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191319 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020573 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155276 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169581 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218029 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182331 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160955 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194123 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488877 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278289 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324895 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347771 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204617 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184859 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197649 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295194 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239228 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269183 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024584 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267097 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387841 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184586 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236775 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017314 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217181 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0182614 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241952 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152087 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263274 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299632 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249266 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240327 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027002 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226794 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292412 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201788 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021982 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210991 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021678 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170324 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018466 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212121 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165163 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224758 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017316 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185847 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182981 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173315 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163264 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158286 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225097 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172246 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319987 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238577 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021925 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280205 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287841 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0172513 s, 3.0 GB/s , 0.0154649 s, 3.4 GB/s 52428800 bytes (52 MB) copied, 0.0173504 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230989 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164242 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167739 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166419 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180782 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225567 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169994 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198523 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176527 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175719 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015678 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022331 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190299 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178031 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200851 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201293 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187707 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179244 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188085 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178007 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202059 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0229108 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180083 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408232 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252192 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219527 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190931 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015712 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017706 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271854 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214835 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173126 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204149 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363162 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225766 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162593 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164665 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157172 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197234 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171361 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181272 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154727 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152768 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178141 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168289 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254837 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172373 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237719 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210494 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197308 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176657 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183303 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210545 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192478 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019117 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161127 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190735 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273476 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242791 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166839 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167062 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241763 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018327 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0155698 s, 3.4 GB/s , 0.015391 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159099 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226507 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176912 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216037 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369101 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212814 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444479 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314929 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337406 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0144215 s, 3.6 GB/s , 0.0157386 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026246 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174502 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271521 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302294 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190885 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198437 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244567 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195956 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229837 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156124 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270733 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037325 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239568 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169769 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201918 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021518 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467862 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047801 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258632 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408379 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275505 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480779 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168899 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352686 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020724 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202555 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220839 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274696 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020194 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292646 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265504 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342551 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276953 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021019 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344973 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337857 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222838 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0645636 s, 812 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04077 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347567 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197641 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322411 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024432 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213917 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271419 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226194 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171997 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183574 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173541 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181882 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224673 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255736 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263948 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033665 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213093 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206656 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329531 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0216401 s, 2.4 GB/s , 0.024546 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229465 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148553 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195961 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158814 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185989 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182926 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355537 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342299 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502913 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292704 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289074 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520312 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239096 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216356 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225201 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253446 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0265767 s, 2.0 GB/s , 0.0268561 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02189 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0144315 s, 3.6 GB/s , 0.0199666 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0189228 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201131 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186409 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237439 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285842 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016915 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184321 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298287 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221783 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157588 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225602 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264769 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298803 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182022 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175599 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015946 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209052 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298394 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210682 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348434 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318082 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017452 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212072 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218318 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213265 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262572 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248239 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187778 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0213774 s, 2.5 GB/s , 0.0208255 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260304 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025051 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231935 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178172 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214375 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161494 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221062 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175898 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208932 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214022 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167384 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197597 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225978 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166826 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224553 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176008 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162298 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356972 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322154 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318006 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181731 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180274 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240025 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176655 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155013 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206682 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195135 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0154081 s, 3.4 GB/s , 0.0194437 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170619 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168706 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156321 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228369 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176285 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205915 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229859 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024405 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226595 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174218 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020972 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0328636 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0179899 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224528 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165172 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179913 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167476 s, 3.1 GB/s , 0.0173541 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203496 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184434 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178499 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0155349 s, 3.4 GB/s , 0.0164769 s, 3.2 GB/s , 0.0189904 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185551 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017942 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266794 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030764 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220338 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439571 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227709 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042453 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188595 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293315 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159391 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220531 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187992 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186247 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432728 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371135 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190977 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303785 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0284166 s, 1.8 GB/s , 0.0244529 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213077 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236539 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292748 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391776 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166696 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179039 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260935 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232337 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381874 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016489 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212144 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346031 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305863 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162418 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186613 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224415 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222475 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186871 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151018 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225793 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157218 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225743 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335906 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02503 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201297 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306777 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155385 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631758 s, 830 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481797 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0378953 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165366 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206937 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207241 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256139 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175374 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017683 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026476 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176504 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201385 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224246 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162286 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192123 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221848 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246607 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359481 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222473 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226659 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217418 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214258 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151703 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180865 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020006 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245374 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204794 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181643 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329974 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253768 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294954 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173567 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213449 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192981 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294509 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179125 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249137 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226371 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196631 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172029 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227437 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268178 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353994 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181629 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187502 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262094 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189682 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212085 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192246 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018154 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187692 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218516 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173097 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249553 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264921 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183963 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017021 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173681 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206045 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178171 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015389 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186009 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169736 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194739 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193751 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151523 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297368 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384919 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378134 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345394 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034433 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017618 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218151 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025433 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211366 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174344 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330168 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374936 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179077 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162255 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213023 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173781 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189689 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214726 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236277 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247728 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213868 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165715 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183876 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264688 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214524 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435415 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045789 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200694 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023061 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196389 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183781 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240735 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194542 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172614 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151761 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442803 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339803 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236686 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0590646 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240764 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242158 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275664 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030925 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177195 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020102 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021513 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199364 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253297 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221297 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172574 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274522 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034623 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326903 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348789 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367595 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307765 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613849 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159158 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493374 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147938 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409785 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262419 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408236 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157071 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272081 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204065 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182819 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185762 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222452 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200654 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280149 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666184 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027671 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483614 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319082 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753612 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022547 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209509 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180036 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021822 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274524 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210384 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191366 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222549 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231845 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178702 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190647 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228891 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266903 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224138 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234786 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286985 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251913 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180891 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202797 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207567 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260256 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170822 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210674 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238601 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218111 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207215 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213601 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215592 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159762 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195823 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174932 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0145648 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173344 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254647 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262484 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029029 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01694 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251315 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183637 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160837 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158867 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232157 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019301 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191862 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330964 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355492 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208967 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243388 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187684 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215849 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280127 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020017 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404835 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148995 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049686 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291802 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212183 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245461 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199348 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175397 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194885 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226615 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155024 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202981 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202834 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168405 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175018 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205303 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022637 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198591 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295855 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213928 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155856 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151593 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154873 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170093 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162981 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247835 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163488 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160732 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180356 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195362 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294548 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222972 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222006 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198542 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290556 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250565 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018028 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166423 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151849 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016117 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158392 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157366 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167276 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214902 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156689 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01923 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203222 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161488 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205335 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229024 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195351 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181002 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204516 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147655 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188338 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236328 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156751 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192808 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173663 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177101 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177037 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016111 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199283 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161973 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183861 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156038 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029148 s, 1.8 GB/s , 0.0282841 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280115 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208836 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172559 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170945 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016047 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169278 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212102 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164555 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203433 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267859 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222346 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017247 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168297 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173372 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155863 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166071 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169992 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198871 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175106 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245131 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183091 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331404 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191046 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215602 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180614 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201513 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163171 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01653 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170936 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154061 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248655 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328829 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396442 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155564 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268574 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223419 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279012 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404042 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337935 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233754 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246159 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249631 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222593 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176147 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260608 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020221 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206173 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206407 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181146 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216295 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190624 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399128 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268354 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188971 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227195 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275143 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296543 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303595 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229814 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249643 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154072 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201534 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231139 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240832 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315966 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420949 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0198663 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253324 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304359 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164222 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232175 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439741 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242157 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179693 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336955 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185475 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243157 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212858 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186882 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234321 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176875 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235856 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198349 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037341 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351236 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282701 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363296 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226803 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284308 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731852 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069932 s, 750 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0209427 s, 2.5 GB/s , 0.0534073 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582249 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239932 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0311252 s, 1.7 GB/s , 0.0382383 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280427 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440128 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428875 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481342 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405918 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201278 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039956 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263767 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206761 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272112 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277768 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167388 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242735 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257343 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230447 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154475 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257946 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183656 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260128 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356844 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250755 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191578 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175149 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213889 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226288 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199542 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179318 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188447 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186044 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203701 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016524 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287611 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021644 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241278 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170029 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226344 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211445 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289297 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209905 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328571 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240637 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257907 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281465 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195456 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189227 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186283 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304376 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506325 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481034 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638864 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220032 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022446 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029332 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228872 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291392 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280574 s, 1.9 GB/s , 0.02081 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298116 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174642 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198517 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294372 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157636 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202591 s, 2.6 GB/s , 0.0202709 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242445 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229849 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183489 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164884 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189343 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402652 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462205 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491875 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373088 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0396058 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299406 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227218 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304495 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208194 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394759 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204844 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205752 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173134 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370615 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177931 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274628 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275592 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398786 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386177 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307939 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346352 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403398 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214678 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334132 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182935 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252603 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302593 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375929 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377327 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213399 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548851 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627666 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578879 s, 906 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247168 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240192 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448872 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245949 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239511 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295228 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292354 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034629 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198424 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363926 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293447 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252517 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185137 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02835 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226039 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252625 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017482 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309666 s, 1.7 GB/s , 0.0298703 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296585 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229572 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286628 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214992 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172316 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393782 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198726 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278391 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158709 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207929 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171126 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254784 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216548 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291642 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336775 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391624 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203741 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300924 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179251 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187707 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161596 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172415 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233313 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239107 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175508 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262952 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211846 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206739 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022598 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022488 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174538 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202758 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207106 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208065 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200279 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258022 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155866 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297674 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417097 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211007 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180207 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251099 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.043061 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209166 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167966 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220164 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177841 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193792 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178751 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233842 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015901 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160525 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169132 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202237 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216977 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160521 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184649 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196418 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0196199 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260076 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159003 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162964 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154483 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260071 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317254 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413408 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03973 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209846 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250738 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312183 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280128 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230102 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223904 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268258 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157196 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017762 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181014 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205637 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260913 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317404 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0158619 s, 3.3 GB/s , 0.0159919 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018928 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194348 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255348 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019112 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208635 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172179 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154532 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167239 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240484 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252552 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164287 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200063 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252002 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021345 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208775 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175618 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279601 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413225 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415704 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436764 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193725 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417331 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449071 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036347 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175643 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193099 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0265901 s, 2.0 GB/s , 0.0200223 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181547 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199589 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201915 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172303 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215695 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153362 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0146938 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196884 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038044 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459074 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251704 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365228 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021215 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148631 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250287 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165501 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161159 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553739 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201673 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158443 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211824 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227422 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251296 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270866 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025726 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174022 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021695 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0169422 s, 3.1 GB/s , 0.0164455 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155774 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176267 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239852 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200986 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252794 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226448 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234308 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186429 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315574 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388945 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571932 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296429 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194027 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253888 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246387 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0265808 s, 2.0 GB/s , 0.0204477 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263536 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348429 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212911 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220385 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400112 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216078 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041737 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411772 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181964 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269496 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190959 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160345 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165628 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151078 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265666 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187252 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287627 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228749 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371875 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399014 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404452 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546021 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361802 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203943 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239456 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224136 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369732 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.044061 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565894 s, 926 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226008 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203462 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215201 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0209922 s, 2.5 GB/s , 0.0184366 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177962 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018822 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184082 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258668 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278934 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251603 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267968 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296061 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243014 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0221185 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171158 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160849 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253999 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208909 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167391 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019629 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298032 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204901 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035112 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194492 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187584 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167593 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220063 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212201 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227708 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253358 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254609 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214993 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227684 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162461 s, 3.2 GB/s 52428800 bytes (52 MB) copied, 0.0190738 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238279 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194765 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227352 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249156 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213933 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343366 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308521 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200714 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288697 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405404 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459614 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230162 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209147 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330621 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209224 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222759 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194947 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159489 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317593 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269691 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162344 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284634 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242022 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199624 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025974 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256662 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233489 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202807 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153997 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210203 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182001 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260065 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171778 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212417 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272512 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255629 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189748 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299714 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310204 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263234 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200359 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220138 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0296632 s, 1.8 GB/s , 0.0232804 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187595 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388214 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303418 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216869 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245045 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223387 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165766 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187535 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267297 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234702 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186093 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166936 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174527 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415308 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424945 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198905 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246563 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295478 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301816 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265225 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231869 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207614 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195421 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219006 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269168 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188811 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217097 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219795 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225916 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029512 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434209 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337249 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364207 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021809 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176326 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020086 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270512 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196954 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226037 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191282 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276799 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282883 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292547 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228385 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238907 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176131 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351271 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288499 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245824 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348088 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394017 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518102 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366155 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03646 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250509 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034377 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195658 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199657 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224118 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203447 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267137 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501871 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171994 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285834 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167451 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193202 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016071 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222169 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270115 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409561 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168865 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246676 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193599 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176966 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184331 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017539 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177231 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172024 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188474 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219509 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196917 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310205 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213461 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279127 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226264 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038915 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292546 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257442 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201695 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212298 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196601 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159689 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298388 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285851 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202319 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175488 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147118 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194989 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224102 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163096 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273346 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021682 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166499 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272212 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208208 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173012 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217751 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214912 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242576 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339392 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476946 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262784 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311961 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241232 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187768 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196758 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224487 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338272 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222031 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265337 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247427 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182067 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203944 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023575 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211976 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240739 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198978 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252919 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239187 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207905 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231749 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339657 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395333 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0444043 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173966 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347618 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341009 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227307 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195989 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177606 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182941 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206452 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301211 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404399 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279616 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188229 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238007 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177766 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306368 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186282 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318846 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351969 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312959 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260402 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154479 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022042 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218642 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337943 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339357 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226763 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277035 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202049 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314941 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186441 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243343 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036727 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253193 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553333 s, 948 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171114 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292031 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035362 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382245 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016331 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355718 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442775 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315243 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223392 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204618 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260202 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273393 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269232 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269036 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407086 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258243 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373219 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323511 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329968 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268403 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039164 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324684 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248123 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212581 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229447 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035063 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378931 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224192 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246231 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352976 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235324 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379414 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225729 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171679 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181253 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029475 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354482 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244627 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038816 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188331 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330828 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371002 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0190844 s, 2.7 GB/s , 0.0232352 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225813 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237039 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224054 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298683 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270494 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186743 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211847 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307816 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237147 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225117 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158616 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248206 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167044 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169473 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167537 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266278 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296072 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300099 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215348 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192378 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020067 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183782 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253909 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255525 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246312 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165084 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182652 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215779 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210907 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372929 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329866 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188151 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288124 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172924 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250772 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168251 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02104 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228754 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203372 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344653 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219468 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377437 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198382 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262664 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180824 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0199301 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292491 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328934 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437631 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251772 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239151 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454945 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185178 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264503 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021548 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021077 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167678 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020596 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352706 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01951 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037739 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328718 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173731 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022042 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296872 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020282 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187797 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254494 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198763 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247285 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206487 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302062 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278705 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218546 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187223 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209059 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240506 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231547 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445019 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304062 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402288 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212172 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251016 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226516 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183566 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196728 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227142 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205688 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296003 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240688 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246701 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231055 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241231 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213464 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169346 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199842 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201938 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281931 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303034 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018283 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173971 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178779 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186107 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193789 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162938 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190448 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215324 s, 2.4 GB/s , 0.0201056 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166519 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195674 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178844 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208446 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252291 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198562 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172237 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019951 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189005 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187055 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02114 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173808 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196636 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187976 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149617 s, 3.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0225881 s, 2.3 GB/s , 0.0178284 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213903 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154739 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023399 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189122 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198779 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218284 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231763 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0180042 s, 2.9 GB/s , 0.0220702 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022108 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180494 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028991 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225225 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020743 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196103 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263432 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203467 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338305 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211243 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324401 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021695 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463577 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333181 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290213 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214652 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032025 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217432 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200091 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196727 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176633 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230444 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341042 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0187588 s, 2.8 GB/s , 0.0279735 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220379 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215541 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272487 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195407 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167356 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171124 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037775 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189702 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250202 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298029 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288613 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249185 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317998 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228517 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286973 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193272 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443495 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599039 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510486 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324764 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0620778 s, 845 MB/s 52428800 bytes (52 MB) copied, 0.0505131 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072355 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381763 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580731 s, 903 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0405436 s, 1.3 GB/s , 0.040046 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307419 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269466 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337101 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167586 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192771 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277134 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235244 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180766 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197858 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020166 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216735 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482323 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210871 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375301 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373553 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043669 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041145 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271592 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361983 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197774 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325434 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215981 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0159794 s, 3.3 GB/s , 0.0166121 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166262 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262666 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018993 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212427 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184307 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021968 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204287 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234715 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278754 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192065 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023588 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284143 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027985 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192823 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255624 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241078 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293924 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229539 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191334 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221314 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192005 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194107 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180315 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164263 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170658 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222249 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183499 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202069 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160457 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159847 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020314 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153727 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167495 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163498 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184229 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197086 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165378 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022157 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171056 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175553 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200216 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320526 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564908 s, 928 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0317475 s, 1.7 GB/s , 0.0308106 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498641 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515696 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233178 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196932 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429439 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198333 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177742 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179141 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023187 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163694 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0159485 s, 3.3 GB/s , 0.0143052 s, 3.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163039 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285831 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192634 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018884 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185861 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244823 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185758 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231917 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358489 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02978 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147615 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332364 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228488 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024964 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334045 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307367 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207402 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167949 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185723 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163523 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192246 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214979 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280914 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.026104 s, 2.0 GB/s , 0.026725 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206237 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391712 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165745 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018912 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248174 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195109 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354896 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028418 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364006 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248975 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279883 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280302 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034029 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201594 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276824 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264424 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187999 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459974 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270981 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260373 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299035 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156991 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227138 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272422 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274272 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179117 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218984 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193653 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264691 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234069 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299299 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187116 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0403718 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161185 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378102 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188479 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275806 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185847 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157421 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181133 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158054 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017347 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0146135 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187098 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228378 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358479 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227608 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271382 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022531 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397753 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257903 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169653 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167084 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167491 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181735 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177266 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027771 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230547 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021134 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176803 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216989 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162105 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172076 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295393 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218041 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276123 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170592 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253458 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232417 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200085 s, 2.6 GB/s , 0.0165186 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222135 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176585 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220442 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302401 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213293 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157005 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0154942 s, 3.4 GB/s , 0.0179001 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016497 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249927 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233111 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219896 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021702 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181923 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227033 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022973 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358311 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257013 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189991 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160177 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274581 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197312 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154163 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257777 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176024 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167646 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155853 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164391 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172998 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020035 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316619 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156525 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195239 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175321 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019862 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019143 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192632 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245365 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348336 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184546 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174899 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215558 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196728 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206987 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017266 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185788 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194401 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186631 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312902 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341217 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309559 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291718 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302233 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022559 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194664 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171477 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292966 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193939 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199778 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368545 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235561 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173833 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026644 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220245 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01866 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0292071 s, 1.8 GB/s , 0.0201254 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211838 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160763 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185878 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227498 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210359 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225954 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022487 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174079 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163067 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163856 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161343 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222316 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161114 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198453 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205615 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179432 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244919 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224415 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191861 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161794 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159164 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305179 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153464 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299549 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214901 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279024 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168033 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156108 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160953 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246157 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220972 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233252 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290239 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354474 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218748 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157222 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330784 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257982 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190651 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215227 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181371 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158433 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209328 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232159 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158211 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232128 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346633 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158939 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207126 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181362 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017399 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190732 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215297 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221697 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298906 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172496 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202983 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180408 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021368 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157338 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275333 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260236 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236037 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230888 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160815 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194005 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252387 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188143 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352808 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184527 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200259 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203431 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166781 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212949 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211339 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198662 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223443 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266592 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239546 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310364 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495783 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455859 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225587 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217091 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176658 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187891 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160479 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254405 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01805 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163071 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259188 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204138 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028691 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310889 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210366 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359571 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049705 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218399 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200807 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015232 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369965 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05497 s, 954 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.022226 s, 2.4 GB/s , 0.0216493 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233796 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240926 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149888 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172701 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165932 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218964 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193328 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226015 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177831 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278628 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371409 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470066 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180014 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222662 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191321 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159909 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193555 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171274 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017045 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197331 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282521 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188982 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343055 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311865 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181688 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231721 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361416 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310836 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208162 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157884 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196455 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225935 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225501 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202428 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204208 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193222 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201727 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176369 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016209 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264746 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242476 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308459 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176361 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224612 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158718 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156518 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180428 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205823 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203495 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181277 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155984 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182891 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232363 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179374 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198528 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176119 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191659 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165486 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02122 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187255 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225753 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211787 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290998 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320786 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213166 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021172 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025772 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302299 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195904 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237715 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307108 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272159 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290091 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250457 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318932 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308856 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388825 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174242 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442428 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182356 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206771 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285668 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246938 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271557 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245132 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021561 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235543 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169388 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225136 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183167 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183905 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256226 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02288 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215432 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199025 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198682 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252214 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204962 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218425 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366193 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230945 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248344 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017232 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315618 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206514 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208616 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244089 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193577 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185028 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166011 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168298 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210647 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180055 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159671 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201529 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316843 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177667 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261939 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199648 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036425 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172987 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290315 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328593 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320737 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294403 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236621 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359147 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152875 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195175 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156236 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323967 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328838 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211435 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0148432 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243781 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352738 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201113 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256647 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020466 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238693 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170016 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203719 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209347 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202622 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019146 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209747 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201456 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198328 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180721 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02424 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159374 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187126 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329799 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0367498 s, 1.4 GB/s , 0.0317196 s, 1.7 GB/s , 0.0448309 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0300513 s, 1.7 GB/s , 0.0160165 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243909 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243952 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311745 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246991 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276807 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298338 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022366 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039175 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0301848 s, 1.7 GB/s , 0.0203796 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219827 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334506 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208214 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184791 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217204 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0308995 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02624 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216215 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350609 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288981 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237547 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394465 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191232 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032577 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390637 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217127 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334565 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276577 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248903 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169861 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243715 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162839 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197783 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234946 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0242164 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387787 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301723 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174262 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381371 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0283604 s, 1.8 GB/s , 0.0186816 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222416 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244014 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216835 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159636 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169497 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208531 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214511 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0175713 s, 3.0 GB/s , 0.0184422 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182571 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352082 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0364384 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471467 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251653 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277814 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021473 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198167 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221559 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198658 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171585 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327152 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026442 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289013 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290569 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233656 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275206 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220695 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356182 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391051 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340927 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342042 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189664 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.022748 s, 2.3 GB/s , 0.0231866 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032697 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171676 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208395 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174969 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016653 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151968 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192934 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187238 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243164 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269938 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351361 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323932 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189487 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431565 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313131 s, 1.7 GB/s , 0.0598536 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160571 s, 3.3 GB/s Finish ======== /mnt/lustre/f44.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x6 lmm_fid: [0x2000059f1:0x6:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x1a4:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800a8819800.stats= snapshot_time 1746143223.111128802 secs.nsecs start_time 1746143162.623637636 secs.nsecs elapsed_time 60.487491166 secs.nsecs read_bytes 902 samples [bytes] 12288 1048576 943718400 989090468069376 read 164650 samples [usecs] 130 322419 87652522 1231714769568 ioctl 1 samples [reqs] open 3294 samples [usecs] 4 73670 3260537 50677328715 close 3294 samples [usecs] 6 32228 1959200 11208382500 seek 3292 samples [usecs] 1 1073 17246 1700952 setattr 1 samples [usecs] 10781 10781 10781 116229961 getattr 17 samples [usecs] 142 7215 34512 193862518 mknod 1 samples [usecs] 17521 17521 17521 306985441 inode_permission 10171 samples [usecs] 1 15148 2334151 9075327689 opencount 3295 samples [reqs] 1 3294 5426866 11919204496 openclosetime 3293 samples [usecs] 3 138718 62455263 2405551400315 pcc_attach 1 samples [reqs] pcc_detach 270 samples [reqs] pcc_auto_attach 270 samples [reqs] pcc_hit_bytes 3275 samples [bytes] 52428800 52428800 171704320000 9002251452416000000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff8800af4f4800.stats= snapshot_time 1746143223.111246025 secs.nsecs start_time 1746143162.623658208 secs.nsecs elapsed_time 60.487587817 secs.nsecs attach 1 detach 270 auto_attach 270 llite.lustre-ffff8800a8819800.pcc_async_threshold=1073741824 llite.lustre-ffff8800af4f4800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d44.sanity-pcc is idle now PASS 44 (65s) == sanity-pcc test 45: Concurrent read access from two mount points ========================================================== 19:47:05 (1746143225) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.152058 s, 690 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d45.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: 561f95fd-8e89-4aad-97e5-f79ec0e25ee6 Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=561f95fd-8e89-4aad-97e5-f79ec0e25ee6 (extents) (64bit) (large files) (huge files) 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.14768 s, 710 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) umount: /mnt/pcc.d45.sanity-pcc.2: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) /tmp/f45.sanity-pcc on /mnt/pcc.d45.sanity-pcc type ext4 (rw,relatime,quota,usrquota,grpquota,data=ordered) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/102400 done Creating filesystem with 102400 1k blocks and 25688 inodes Filesystem UUID: b273543b-c5c4-4da3-8ebb-ffbd9af1a80a Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/13 done Writing inode tables: 0/13 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/13 done /tmp/f45.sanity-pcc.2: Linux rev 1.0 ext4 filesystem data, UUID=b273543b-c5c4-4da3-8ebb-ffbd9af1a80a (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d45.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} pcc: - pccpath: /mnt/pcc.d45.sanity-pcc.2 hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8819800.pcc_async_threshold=0 llite.lustre-ffff8800af4f4800.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.403871 s, 130 MB/s /mnt/lustre/f45.sanity-pcc lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 421 0x1a5 0x280000400 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.321751 s, 163 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.459801 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.460632 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.436361 s, 120 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.441562 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.440611 s, 119 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.373139 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.474597 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.44472 s, 118 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.468704 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.471827 s, 111 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.478404 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.490757 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.509639 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.496429 s, 106 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.530879 s, 98.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.504105 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.505854 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.50681 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.508633 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.50942 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.509982 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.511976 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.512222 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.514004 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.514821 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.516776 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.519103 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.519497 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.523183 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.526882 s, 99.5 MB/s 52428800 bytes (52 MB) copied, 0.527011 s, 99.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228495 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519386 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0411517 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554582 s, 945 MB/s 52428800 bytes (52 MB) copied, 0.0465122 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496537 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395123 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789934 s, 664 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482063 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412649 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318917 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527733 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682439 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644172 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589242 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067072 s, 782 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522905 s, 1.0 GB/s , 0.0570894 s, 918 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497876 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198174 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736343 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349791 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331131 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372474 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633743 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258324 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322234 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407811 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202093 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467041 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307361 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244866 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327244 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0614143 s, 854 MB/s 52428800 bytes (52 MB) copied, 0.051992 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026841 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615496 s, 852 MB/s 52428800 bytes (52 MB) copied, 0.058733 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397144 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222059 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256286 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175984 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212935 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201698 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254339 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186465 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195179 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476024 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388946 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411408 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411743 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260455 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053966 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428584 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390446 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0475143 s, 1.1 GB/s , 0.0528892 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415585 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377609 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300554 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0470332 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0544373 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321278 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229906 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200297 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316547 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024446 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253188 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239007 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0184008 s, 2.8 GB/s , 0.0220668 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205596 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259793 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178395 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221057 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185318 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168716 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185703 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180589 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209763 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160102 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172677 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338145 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017743 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356002 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369151 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193064 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340369 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029261 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375854 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342424 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276862 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206078 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233225 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308785 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162235 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193644 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304185 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174737 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186754 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275898 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215471 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477304 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03508 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260258 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242817 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322099 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207112 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184155 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224945 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225428 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170907 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206507 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222945 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211744 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273243 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241129 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248769 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344054 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440429 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311999 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278025 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186255 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267984 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217514 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250074 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021985 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183191 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229058 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156618 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195596 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228117 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210981 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277389 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177355 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174624 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257641 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344389 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019953 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223979 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01894 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176324 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015817 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210226 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286999 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0359675 s, 1.5 GB/s , 0.0495107 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413698 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529849 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486257 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483378 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356666 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036907 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326479 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357938 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528059 s, 993 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018352 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387133 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0422222 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220944 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299862 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288729 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265548 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270162 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269233 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209348 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207505 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267609 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226138 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237192 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024721 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224911 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190724 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201037 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230969 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0222358 s, 2.4 GB/s , 0.0289024 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482971 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244685 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323332 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382462 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436293 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416476 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.043311 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586982 s, 893 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358519 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477972 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888822 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505307 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227222 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332282 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276087 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356933 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043643 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379645 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218392 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197257 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0471853 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403774 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186874 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285721 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276796 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418132 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246746 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241143 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419771 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0369985 s, 1.4 GB/s 50+0 records in 50+0 records out , 0.0263205 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0496924 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290205 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247582 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233328 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301822 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196449 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345103 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268112 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302674 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202015 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346571 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230838 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029062 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318266 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364628 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026979 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283602 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364228 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213272 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359482 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308877 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292945 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281207 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165605 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269156 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226623 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216399 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234281 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472405 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432175 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283253 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410048 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313836 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253403 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254166 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258922 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248868 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231532 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035776 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348548 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406738 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390928 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268657 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025975 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287293 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385512 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233713 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626103 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395541 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732398 s, 716 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441999 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410159 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517493 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0869527 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043075 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526956 s, 995 MB/s 52428800 bytes (52 MB) copied, 0.0541369 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641995 s, 817 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697665 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0593887 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753291 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797937 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696973 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494803 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451276 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797134 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261572 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026534 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522531 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323623 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245383 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410356 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392396 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309679 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203179 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221511 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020238 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224739 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321907 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030364 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283903 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211336 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271882 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248464 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183616 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358612 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0376076 s, 1.4 GB/s , 0.0232534 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193253 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201656 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254845 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240838 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195843 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02308 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267775 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191187 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444944 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328941 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210753 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227165 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018789 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198992 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172242 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255777 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206308 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315525 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264738 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231211 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343983 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348638 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350033 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198994 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164888 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175059 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174965 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271762 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153919 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187405 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180547 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189136 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198408 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176171 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173183 s, 3.0 GB/s 52428800 bytes (52 MB) copied, 0.0295777 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208481 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167361 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279823 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241761 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397571 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413286 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370939 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753234 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420719 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350676 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.05581 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365076 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383503 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614098 s, 854 MB/s 52428800 bytes (52 MB) copied, 0.0371165 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189815 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210039 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199836 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195686 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177172 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017711 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197239 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020893 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185955 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224428 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040992 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278986 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477002 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525692 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182056 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0644105 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355685 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256522 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359084 s, 1.5 GB/s , 0.0415664 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441591 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341504 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274969 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397025 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0563808 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342388 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052038 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250961 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606603 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736335 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034557 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021893 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203382 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269137 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228865 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186915 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171799 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150761 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159411 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164967 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217097 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175674 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225707 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197483 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193521 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279252 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234972 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194009 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157843 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156798 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019702 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186927 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198644 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024788 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248107 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027382 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261966 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029518 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211187 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273321 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174792 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162777 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163688 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196153 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209498 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252272 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021551 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202498 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194703 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182459 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201662 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162817 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172202 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237599 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327854 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298299 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237632 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298798 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334222 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246135 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271489 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017921 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219692 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171801 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213996 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179529 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358809 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198182 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237657 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382217 s, 1.4 GB/s , 0.0301974 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351459 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200263 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454832 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419606 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364036 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420069 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261685 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176233 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222645 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351412 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200072 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261092 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252462 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246453 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324205 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297695 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043722 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436211 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219845 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285092 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306896 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372848 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033708 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391101 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693229 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412928 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0829336 s, 632 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0939922 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749038 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726543 s, 722 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107795 s, 486 MB/s 52428800 bytes (52 MB) copied, 0.0679892 s, 771 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0880253 s, 596 MB/s , 0.10386 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882074 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0788033 s, 665 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.096082 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770075 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0958309 s, 547 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050717 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.080495 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607582 s, 863 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423845 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0906163 s, 579 MB/s 52428800 bytes (52 MB) copied, 0.0320857 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12317 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413328 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056819 s, 923 MB/s 52428800 bytes (52 MB) copied, 0.0377913 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442519 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374392 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657136 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397298 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052337 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281798 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230294 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448417 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233268 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254699 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181334 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180202 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240012 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251093 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238258 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229687 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201229 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200722 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194773 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024476 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199817 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016083 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238448 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184314 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207035 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414952 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599372 s, 875 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349195 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0239408 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756479 s, 693 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402001 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352126 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349161 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305471 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268962 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498636 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0518663 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263094 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450266 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270411 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258845 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.101165 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0897569 s, 584 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481155 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0991492 s, 529 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864035 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398032 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394467 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447677 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290586 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375475 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0602522 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269837 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180145 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016561 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193403 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.031722 s, 1.7 GB/s , 0.0239768 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207779 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225581 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319606 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215988 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380204 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0170322 s, 3.1 GB/s , 0.0346277 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214574 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301324 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237432 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220014 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184511 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168867 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200288 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0158698 s, 3.3 GB/s , 0.0247075 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181278 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178864 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234273 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270548 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292447 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180005 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016086 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173046 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186514 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208968 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197985 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275148 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196107 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165383 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219133 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197334 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265691 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386959 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222254 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237765 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308585 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207477 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541748 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184929 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618802 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240687 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177334 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169643 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035807 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.043949 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437416 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512054 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483016 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572643 s, 916 MB/s 52428800 bytes (52 MB) copied, 0.0405625 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313501 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0467869 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038723 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568082 s, 923 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224926 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241401 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200419 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196838 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0284743 s, 1.8 GB/s , 0.0191508 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275932 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038525 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201655 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179359 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197713 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183621 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183656 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241488 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332676 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205303 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268418 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222889 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189462 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016018 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245895 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228395 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212864 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194491 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220085 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258031 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291944 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204203 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0262834 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255558 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320033 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039587 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250707 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0275402 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427703 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037616 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261369 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218924 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174366 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196827 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204527 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177634 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175371 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285905 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239775 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0302118 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303332 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211207 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392503 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180628 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179203 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229492 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193338 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273174 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192856 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021534 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227531 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236142 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247728 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025865 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184938 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047646 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214716 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187348 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306396 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249931 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322074 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212653 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171295 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194953 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0249532 s, 2.1 GB/s , 0.0172936 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320245 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186516 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242135 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204171 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167221 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168795 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206765 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237881 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0398011 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330275 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336405 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209185 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380078 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501715 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384523 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470342 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393692 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0336867 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407287 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270604 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415058 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190179 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174154 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229162 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171015 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179961 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181123 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184839 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211962 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173383 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026981 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271251 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241839 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0326961 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313991 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366394 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528791 s, 991 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306882 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556831 s, 942 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323157 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270678 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610328 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306942 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453542 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234935 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242001 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289562 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202165 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018335 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268053 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164974 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513168 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174604 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213599 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216801 s, 2.4 GB/s , 0.0417466 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234379 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179218 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023316 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318988 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029734 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018929 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05024 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335835 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274609 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320605 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240755 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210109 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284434 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215978 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191803 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254625 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202392 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338519 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390144 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052587 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286275 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423788 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812165 s, 646 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06417 s, 817 MB/s 52428800 bytes (52 MB) copied, 0.0642532 s, 816 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560205 s, 936 MB/s 52428800 bytes (52 MB) copied, 0.050695 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677653 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353358 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347409 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547602 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199633 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501807 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460052 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365457 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166086 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270893 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199694 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399459 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363456 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225231 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0260598 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0411647 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040115 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228675 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375453 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171675 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273637 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333611 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291316 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268094 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280673 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367656 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270313 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204784 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226399 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262453 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359927 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251661 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248679 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207378 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223955 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223625 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230239 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337925 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319185 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320202 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272315 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272978 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613332 s, 855 MB/s 52428800 bytes (52 MB) copied, 0.0546063 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448571 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610771 s, 858 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0624102 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433496 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761736 s, 688 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875395 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061392 s, 854 MB/s 52428800 bytes (52 MB) copied, 0.100824 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.096601 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0635105 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795788 s, 659 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511208 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698083 s, 751 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045111 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429973 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0464272 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061343 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0636806 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612723 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348412 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417629 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072885 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235213 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272618 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249739 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313991 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411754 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022723 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.06956 s, 754 MB/s 52428800 bytes (52 MB) copied, 0.0733934 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182762 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0616387 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188483 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101043 s, 519 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428842 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060943 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04606 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291007 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474088 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030556 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567126 s, 924 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307135 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221348 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024261 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328843 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395968 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438857 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041829 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257918 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296873 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772682 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411831 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572344 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367144 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348026 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447326 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404416 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474948 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586029 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033164 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828109 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053003 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.071082 s, 738 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776907 s, 675 MB/s 52428800 bytes (52 MB) copied, 0.0609721 s, 860 MB/s 52428800 bytes (52 MB) copied, 0.0596101 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798151 s, 657 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742072 s, 707 MB/s 52428800 bytes (52 MB) copied, 0.0965764 s, 543 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111382 s, 471 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176567 s, 297 MB/s 52428800 bytes (52 MB) copied, 0.178265 s, 294 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176766 s, 297 MB/s 52428800 bytes (52 MB) copied, 0.151034 s, 347 MB/s 52428800 bytes (52 MB) copied, 0.166294 s, 315 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153805 s, 341 MB/s 52428800 bytes (52 MB) copied, 0.168727 s, 311 MB/s 52428800 bytes (52 MB) copied, 0.192159 s, 273 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177331 s, 296 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135325 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.204465 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151202 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142661 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164299 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.216709 s, 242 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.172863 s, 303 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.185774 s, 282 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.232244 s, 226 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0935826 s, 560 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120448 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182305 s, 288 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.129462 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199213 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484985 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700222 s, 749 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720312 s, 728 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501024 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419704 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705876 s, 743 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217472 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230336 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179495 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293523 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032084 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344938 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0569081 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427487 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346818 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0804769 s, 651 MB/s 52428800 bytes (52 MB) copied, 0.0687395 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396969 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306616 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506756 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237039 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193726 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237956 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388727 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215575 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425561 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275632 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440167 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267664 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435173 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212232 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039864 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408811 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420133 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388951 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0671642 s, 781 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254326 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594021 s, 883 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0611784 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494485 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820797 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.072543 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517304 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329367 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310754 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240239 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186116 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180586 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184043 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298595 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280874 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205004 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240029 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022538 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355409 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307645 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221615 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297151 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329549 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251052 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175091 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299399 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359662 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0495903 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0746002 s, 703 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0846839 s, 619 MB/s 52428800 bytes (52 MB) copied, 0.085747 s, 611 MB/s 52428800 bytes (52 MB) copied, 0.0800988 s, 655 MB/s 52428800 bytes (52 MB) copied, 0.0913757 s, 574 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830623 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105293 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116063 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948687 s, 553 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128508 s, 408 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121409 s, 432 MB/s 52428800 bytes (52 MB) copied, 0.0865443 s, 606 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539523 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13638 s, 384 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0697293 s, 752 MB/s 52428800 bytes (52 MB) copied, 0.0905916 s, 579 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353295 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394523 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0699188 s, 750 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501071 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260134 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297227 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188259 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0697464 s, 752 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213247 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336459 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357102 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022612 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020547 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241233 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313808 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408873 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166163 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283084 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254042 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172304 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180427 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201372 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209239 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284028 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283338 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203489 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345165 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182603 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204266 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192118 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318646 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346995 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379738 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364444 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300197 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299413 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456849 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455015 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066489 s, 789 MB/s 52428800 bytes (52 MB) copied, 0.0494 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602306 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637792 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0667425 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460104 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695065 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700537 s, 748 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0388863 s, 1.3 GB/s , 0.0783283 s, 669 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0696562 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634564 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03975 s, 1.3 GB/s , 0.056991 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387027 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310162 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352172 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213163 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191087 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167066 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313422 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030108 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440386 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209437 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267087 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307432 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456852 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254523 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394583 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339485 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401019 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0456461 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311415 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028805 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503478 s, 1.0 GB/s , 0.0349309 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506195 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390066 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262894 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276203 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487207 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030239 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217202 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442754 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258648 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327032 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275731 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226659 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444233 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274401 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209283 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458843 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333718 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481446 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304968 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247732 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200357 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161122 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029931 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266466 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284632 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206032 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457272 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360643 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494334 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480506 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0581181 s, 902 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05826 s, 900 MB/s 52428800 bytes (52 MB) copied, 0.0974897 s, 538 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484887 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0793655 s, 661 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536258 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919791 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128906 s, 407 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485335 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693631 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107354 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116989 s, 448 MB/s , 0.0533923 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0858401 s, 611 MB/s 52428800 bytes (52 MB) copied, 0.0680672 s, 770 MB/s 50+0 records in 50+0 records out , 0.0811426 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588065 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115183 s, 455 MB/s 52428800 bytes (52 MB) copied, 0.110437 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816557 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0643685 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607794 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0807512 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307379 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294219 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025072 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284217 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192192 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243814 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204312 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193872 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210314 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259376 s, 2.0 GB/s , 0.0267526 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239288 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238455 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032197 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371253 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249691 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256701 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194855 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0234382 s, 2.2 GB/s , 0.0276822 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183251 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180022 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209193 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259661 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247857 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297122 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182893 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157539 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182831 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203349 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223751 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184614 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018317 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019643 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.035768 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195971 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180551 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193113 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172414 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180977 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193019 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171631 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238428 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163417 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167057 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231301 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393872 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267438 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019799 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223205 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191721 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180681 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0234113 s, 2.2 GB/s , 0.025621 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363734 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170808 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365291 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024239 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243183 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034392 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308732 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030232 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190655 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242954 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258021 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177215 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290093 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189178 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334474 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165101 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434802 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627632 s, 835 MB/s 52428800 bytes (52 MB) copied, 0.0570313 s, 919 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494017 s, 1.1 GB/s 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0564517 s, 929 MB/s , 0.0402052 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300207 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0601109 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316675 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436836 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0761561 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337803 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323563 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217715 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289883 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165616 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018281 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161645 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263516 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216365 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190187 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231042 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196312 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158598 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154407 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244746 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205765 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184087 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221964 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170979 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216574 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323967 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395917 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430653 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254078 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339434 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628069 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427825 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278494 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378284 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220704 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396814 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194102 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0337461 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479072 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325303 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183333 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294084 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314121 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021526 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0476726 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185903 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242595 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281498 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175321 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226367 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211313 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213622 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193736 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293414 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194686 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270106 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206979 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334372 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156691 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186814 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179583 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176977 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199791 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221784 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206816 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451364 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774985 s, 677 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731293 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471228 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0580732 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706671 s, 742 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695338 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344337 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456067 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548793 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297587 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308132 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461648 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449486 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573002 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387549 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226102 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245018 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573237 s, 915 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293651 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0574021 s, 913 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176475 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220205 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249444 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269953 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274203 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261297 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406598 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0194192 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171911 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292078 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153432 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161696 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191986 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226203 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189121 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192887 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304518 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242392 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018902 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290185 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0391589 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025632 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165347 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207095 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266911 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174643 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384942 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296327 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203778 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333276 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0281701 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231995 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181907 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0269007 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0165854 s, 3.2 GB/s , 0.0168003 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200474 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0359047 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176891 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408415 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326828 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557087 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351735 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232699 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206052 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327562 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322161 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198114 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247159 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155382 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022407 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165482 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656095 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287522 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216056 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345103 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183371 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383911 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551636 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181966 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405687 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189514 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020459 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170528 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241639 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281618 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191346 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277933 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211591 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186705 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189586 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02517 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196052 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275551 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237829 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223542 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181612 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174638 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018063 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295529 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234624 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235941 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225765 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167347 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259585 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356209 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205923 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162634 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189981 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201282 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161458 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151476 s, 3.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0186646 s, 2.8 GB/s , 0.0150262 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244447 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207534 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214775 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180535 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192016 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188283 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207222 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236395 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359972 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185378 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244416 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181847 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336509 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239208 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326798 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348309 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223434 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0631801 s, 830 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547358 s, 958 MB/s 52428800 bytes (52 MB) copied, 0.0544114 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603185 s, 869 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604801 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552681 s, 949 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450284 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639214 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492104 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458996 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473418 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458208 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609537 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364094 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511201 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472037 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403469 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603783 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291217 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347203 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273526 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231767 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173466 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232428 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372009 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183761 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034237 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166642 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207673 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191666 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276849 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282081 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321904 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0379021 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185651 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312927 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304501 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243278 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215011 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017945 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178159 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177534 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174935 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181458 s, 2.9 GB/s , 0.0187194 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197634 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177535 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173359 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189276 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180857 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201864 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181852 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187912 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187505 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196483 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322043 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256436 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201433 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275238 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281499 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206506 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292107 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024726 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274126 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379601 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474566 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370098 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042898 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422861 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490285 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383374 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189857 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324501 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204387 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189908 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218623 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226531 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217662 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234598 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267964 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286393 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190609 s, 2.8 GB/s , 0.0190179 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416585 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208009 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178002 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347615 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266514 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192691 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182184 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428116 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0412045 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394033 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374934 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376781 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0502765 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318174 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061982 s, 846 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535588 s, 979 MB/s 52428800 bytes (52 MB) copied, 0.0693955 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0550487 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409901 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463506 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537139 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037021 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336255 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286887 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334861 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022196 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453499 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178469 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175681 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343897 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215043 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261712 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221551 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253438 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184365 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170873 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026966 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250428 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215689 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212278 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0168374 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256487 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236146 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202942 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184351 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253264 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165116 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211272 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191774 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279672 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369634 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292883 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0448686 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637281 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276759 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704853 s, 744 MB/s 52428800 bytes (52 MB) copied, 0.0743103 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618314 s, 848 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0861472 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779732 s, 672 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598092 s, 877 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0705988 s, 743 MB/s 52428800 bytes (52 MB) copied, 0.062459 s, 839 MB/s 52428800 bytes (52 MB) copied, 0.0656542 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0661722 s, 792 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609042 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062691 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565158 s, 928 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045078 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584805 s, 897 MB/s 52428800 bytes (52 MB) copied, 0.0397708 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510725 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263524 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417117 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213609 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233131 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176706 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219876 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183167 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166512 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175143 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247783 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195762 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0335794 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270966 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167274 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022621 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193863 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251912 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016125 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192688 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179511 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153232 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396732 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210789 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261892 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182773 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229127 s, 2.3 GB/s , 0.0383352 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426034 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040856 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393014 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311821 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191453 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332393 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439885 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183244 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177848 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230815 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267062 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260392 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285272 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248945 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315743 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0351309 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304641 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231386 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235443 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332515 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0150635 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195463 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163604 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233273 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200852 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021023 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210677 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203717 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190652 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213453 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219581 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234151 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302058 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349033 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281318 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373921 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666668 s, 786 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300316 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0581922 s, 901 MB/s 52428800 bytes (52 MB) copied, 0.0463614 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0347593 s, 1.5 GB/s , 0.0217595 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189567 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024624 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240801 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291298 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189737 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198992 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207087 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177329 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183403 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215804 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192674 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167136 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199755 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222899 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028389 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204042 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197334 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237563 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185943 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176309 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169256 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204296 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248189 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197561 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285284 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464787 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403713 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0469413 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462129 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393036 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408416 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0575865 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304051 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0784754 s, 668 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423811 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512777 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763576 s, 687 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0749858 s, 699 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244704 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371847 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216254 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184005 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225645 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232163 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218641 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244293 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179919 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023577 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342802 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316935 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327922 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437204 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387219 s, 1.4 GB/s , 0.0280519 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450528 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308981 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296492 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321691 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344278 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255736 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163978 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017395 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257118 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228375 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262507 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210539 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202178 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200681 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372204 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352103 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0318366 s, 1.6 GB/s , 0.0246525 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454463 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506854 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276228 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806535 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0648522 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449707 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397816 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352898 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434916 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437812 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270074 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457456 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168545 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263902 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257526 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262879 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354134 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207848 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410521 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0270806 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198412 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299668 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178117 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606687 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487736 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428136 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248582 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352492 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200854 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260461 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207484 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218915 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218081 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040585 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234109 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165904 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406413 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170192 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222105 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232412 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181507 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426748 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434448 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164537 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170596 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360523 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323443 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170405 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419942 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015763 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248902 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199739 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256406 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163525 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199737 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327907 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472646 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360847 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368149 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491214 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209793 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0539735 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430742 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103039 s, 509 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0506524 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191392 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885437 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767038 s, 684 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288382 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155329 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203028 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017778 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028154 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221176 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252716 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384297 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240279 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207467 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160748 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260965 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237907 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285398 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.046349 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553792 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567948 s, 923 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.096722 s, 542 MB/s 52428800 bytes (52 MB) copied, 0.0475981 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653014 s, 803 MB/s 52428800 bytes (52 MB) copied, 0.0731626 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524644 s, 999 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101496 s, 517 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651603 s, 805 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755064 s, 694 MB/s 52428800 bytes (52 MB) copied, 0.0575548 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711171 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446682 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522911 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118491 s, 442 MB/s 52428800 bytes (52 MB) copied, 0.0444563 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576023 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0743 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120064 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0956347 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608313 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637401 s, 823 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190441 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315688 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505365 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0308858 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277678 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460225 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221915 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479502 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496194 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0761777 s, 688 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267858 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440465 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.042518 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0271128 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162567 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177833 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0455687 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214563 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210652 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310478 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453499 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157573 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363387 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026144 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400252 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216089 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401788 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183554 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310685 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247124 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436017 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0459818 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017813 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0403667 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192919 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185641 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444523 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333495 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296306 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256037 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253538 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217001 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152052 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232079 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266625 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197019 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293912 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209397 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017369 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016548 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024341 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162145 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471158 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233068 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264384 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365418 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451482 s, 1.2 GB/s , 0.0205505 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344315 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430473 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0968612 s, 541 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532931 s, 984 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675458 s, 776 MB/s 52428800 bytes (52 MB) copied, 0.0556768 s, 942 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0757488 s, 692 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432337 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08014 s, 654 MB/s 52428800 bytes (52 MB) copied, 0.125105 s, 419 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13548 s, 387 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497785 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0467333 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580504 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236612 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273701 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304024 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453517 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256687 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.021605 s, 2.4 GB/s , 0.0196608 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258807 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178138 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280415 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276535 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024294 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215217 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298698 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213948 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261716 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388204 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214698 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261413 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0373514 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047152 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350066 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433527 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040125 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348338 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0687826 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627135 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585665 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035934 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225707 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174691 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169658 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174586 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189994 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181857 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434733 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319228 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336606 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304507 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320098 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417675 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0349161 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232089 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304432 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037899 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480257 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0228987 s, 2.3 GB/s , 0.024077 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277636 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230063 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293659 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473967 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487375 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398468 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0582554 s, 900 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377981 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601122 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421302 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384942 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581662 s, 901 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306888 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209671 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191854 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223511 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168797 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314917 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210507 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672805 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517987 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045627 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695247 s, 754 MB/s , 0.0498923 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594818 s, 881 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598114 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279871 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472017 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255596 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633266 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519177 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454362 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0576195 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426775 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163933 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225782 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204121 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232374 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316877 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312728 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246802 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401352 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292701 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259989 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252202 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233441 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254203 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491268 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399596 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314598 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183718 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293329 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280631 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018094 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.024435 s, 2.1 GB/s , 0.0227827 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298931 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257614 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225778 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259742 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017322 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358746 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208386 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373097 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168643 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327261 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344524 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273875 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207366 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183177 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276827 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437642 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452778 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312958 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291435 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265351 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421156 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054336 s, 965 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879129 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500462 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103968 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553404 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061439 s, 853 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716197 s, 732 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0787382 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923806 s, 568 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0939064 s, 558 MB/s 52428800 bytes (52 MB) copied, 0.0710732 s, 738 MB/s 52428800 bytes (52 MB) copied, 0.0904519 s, 580 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701058 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731873 s, 716 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114389 s, 458 MB/s 52428800 bytes (52 MB) copied, 0.049549 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542558 s, 966 MB/s 52428800 bytes (52 MB) copied, 0.043751 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0987584 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353539 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469308 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513564 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.096258 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242339 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606788 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052544 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487112 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203611 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292955 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018825 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033674 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409735 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053301 s, 984 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279241 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535391 s, 979 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326556 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.050144 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0886129 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523331 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422179 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0807735 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435473 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039288 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521443 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0503797 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415668 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294742 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053617 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315036 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039154 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0949765 s, 552 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894715 s, 586 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0888542 s, 590 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601682 s, 871 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0970298 s, 540 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892337 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412137 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102752 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.086087 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.126545 s, 414 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110617 s, 474 MB/s 52428800 bytes (52 MB) copied, 0.0504959 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118377 s, 443 MB/s , 0.0495484 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.121563 s, 431 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0910809 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0588604 s, 891 MB/s 52428800 bytes (52 MB) copied, 0.120388 s, 436 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0688506 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469326 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274123 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410054 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035197 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0810359 s, 647 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.073156 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034833 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214236 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075127 s, 698 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381617 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0357928 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816141 s, 642 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028491 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0844061 s, 621 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549503 s, 954 MB/s , 0.118645 s, 442 MB/s 52428800 bytes (52 MB) copied, 0.150415 s, 349 MB/s 52428800 bytes (52 MB) copied, 0.0860235 s, 609 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0964862 s, 543 MB/s , 0.0696612 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847883 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228074 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211104 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497333 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386458 s, 1.4 GB/s , 0.067951 s, 772 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368088 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275472 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561185 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225227 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470646 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104459 s, 502 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452958 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104956 s, 500 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102938 s, 509 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503729 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0456274 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619683 s, 846 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396211 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0784252 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523767 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289991 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382981 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040353 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516833 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478541 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0435505 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341041 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300613 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026306 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0821036 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845494 s, 620 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412144 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0596576 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144727 s, 362 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111767 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147247 s, 356 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539657 s, 972 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0630913 s, 831 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0565262 s, 928 MB/s 52428800 bytes (52 MB) copied, 0.111852 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098791 s, 531 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109379 s, 479 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342758 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0783937 s, 669 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499691 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540462 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029081 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608015 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0998024 s, 525 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473812 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0909923 s, 576 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386424 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432513 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400322 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334531 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296446 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498103 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412534 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384357 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0548011 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311536 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367107 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567926 s, 923 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10759 s, 487 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568739 s, 922 MB/s 52428800 bytes (52 MB) copied, 0.0702543 s, 746 MB/s 52428800 bytes (52 MB) copied, 0.0648679 s, 808 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355747 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406485 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029293 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369387 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198501 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01843 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0141497 s, 3.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021116 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169039 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337754 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170314 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336297 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310403 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111055 s, 472 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315917 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619353 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019516 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0400269 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242318 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187801 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347342 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206499 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263094 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190374 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250554 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019333 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169666 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197588 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0331303 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451822 s, 1.2 GB/s , 0.0406155 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267814 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348972 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314176 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303654 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348978 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023314 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033819 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311261 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022864 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326143 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382985 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05051 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0563459 s, 930 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417898 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483725 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530807 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503834 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0762497 s, 688 MB/s 52428800 bytes (52 MB) copied, 0.0603978 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673131 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471581 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0706272 s, 742 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653612 s, 802 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063435 s, 826 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288693 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.043908 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223377 s, 2.3 GB/s , 0.0355292 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376241 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200288 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191762 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191186 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286275 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217822 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251864 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200101 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224477 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262358 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254203 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187579 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0171457 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0204773 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213229 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.01988 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190329 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194296 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372846 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205889 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349809 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254287 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284107 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0403506 s, 1.3 GB/s , 0.0302574 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465277 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365675 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172115 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158655 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188076 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293203 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229546 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410922 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0269079 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397663 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201413 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239431 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174227 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177939 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197732 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189249 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159891 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180624 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168088 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178394 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210189 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020342 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299492 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0347203 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355196 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285119 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164556 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217412 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265813 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165441 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184241 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268906 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0274204 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347851 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192002 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190736 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259874 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209064 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201691 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328879 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293546 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475457 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.036738 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211113 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433899 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461267 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0330412 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277038 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260996 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294845 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200352 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238665 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263471 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021589 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194263 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284515 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388373 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293386 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332635 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256758 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275477 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589929 s, 889 MB/s , 0.0290477 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334061 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185466 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022345 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220163 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216538 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197223 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255037 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155294 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173573 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189998 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175442 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203036 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0250589 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0201997 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017618 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252797 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160811 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156878 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186239 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016023 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169403 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024707 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319367 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320103 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303954 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195915 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177173 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218716 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428595 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394641 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019884 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0281147 s, 1.9 GB/s , 0.0161163 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0178027 s, 2.9 GB/s , 0.0220356 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180356 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020272 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224622 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193864 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194026 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206489 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307681 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017201 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228293 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207047 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016402 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176304 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204066 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262635 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334356 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213153 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220103 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249417 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022232 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191821 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021692 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306298 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232955 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209903 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212256 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238552 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305201 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293728 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191064 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300148 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021033 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400849 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464417 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032681 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314872 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178202 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211034 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166906 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237428 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203813 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196505 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016215 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176693 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212061 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195687 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208364 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233797 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200494 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447884 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227817 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029703 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309014 s, 1.7 GB/s , 0.019378 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250163 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306232 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313566 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536455 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499258 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418446 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445341 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0786037 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549902 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586755 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.068277 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0807047 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974259 s, 538 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734627 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527103 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575308 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390608 s, 1.3 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381871 s, 1.4 GB/s , 0.0688126 s, 762 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442815 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356915 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291833 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237238 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182277 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173741 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020275 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235899 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348863 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022915 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193181 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216451 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179362 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186107 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184347 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268409 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202026 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184834 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185981 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237846 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224329 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195081 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226996 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0233889 s, 2.2 GB/s , 0.0175388 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220774 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019074 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232495 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0195315 s, 2.7 GB/s , 0.0220404 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209787 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179937 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170097 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166834 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228215 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199657 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220389 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212704 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189695 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385089 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368692 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338292 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479654 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029181 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505521 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364076 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468871 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230467 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259157 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214072 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0187891 s, 2.8 GB/s , 0.0229252 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385459 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427519 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147535 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207089 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224009 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190495 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184976 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184264 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252701 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187795 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232118 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175314 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329704 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289164 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218004 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223662 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212997 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237512 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334093 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253529 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342311 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441632 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402346 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508243 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0756083 s, 693 MB/s 52428800 bytes (52 MB) copied, 0.0522369 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549828 s, 954 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466755 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0657832 s, 797 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0774238 s, 677 MB/s 52428800 bytes (52 MB) copied, 0.0538229 s, 974 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026977 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625942 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557306 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084364 s, 621 MB/s 52428800 bytes (52 MB) copied, 0.0826767 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595469 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047269 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217794 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395104 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549166 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514813 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169911 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203684 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02097 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026009 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435574 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220275 s, 2.4 GB/s , 0.0223099 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186402 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190498 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331741 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345267 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179303 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167267 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263181 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280089 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304904 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322776 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0790592 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256823 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379262 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353528 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200467 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020469 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240374 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389247 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184601 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235127 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193288 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324697 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261781 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021416 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182465 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192395 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023016 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161064 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188129 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174831 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175588 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187212 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04128 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173037 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214955 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238988 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206479 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275693 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287048 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350447 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322737 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241822 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180338 s, 2.9 GB/s 52428800 bytes (52 MB) copied, 0.0322575 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192614 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299689 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209405 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369592 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033294 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186173 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198627 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327235 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403944 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540087 s, 971 MB/s 52428800 bytes (52 MB) copied, 0.0298813 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357738 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0798504 s, 657 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322364 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362442 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244957 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063398 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414207 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232736 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208638 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475685 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476262 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1175 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266712 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202248 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208054 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019811 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171067 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167921 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210766 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181782 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181454 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255533 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0187659 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0252299 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214801 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020789 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169481 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161319 s, 3.3 GB/s , 0.0247226 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177315 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235581 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197859 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178824 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214685 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181531 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242816 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295963 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418411 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0591695 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220295 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.053778 s, 975 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467467 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418331 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551767 s, 950 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487038 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027137 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182326 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205116 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253294 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0303912 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401307 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0249496 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231241 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218054 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271595 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187286 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185194 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216949 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182955 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274257 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232811 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191581 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157557 s, 3.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0193067 s, 2.7 GB/s , 0.0158699 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178411 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205727 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211354 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224593 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196158 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194761 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195504 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190608 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174334 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170365 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239769 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195105 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187794 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237357 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0367749 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259049 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336036 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334805 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218483 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018134 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303541 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017377 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176433 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187428 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179156 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186216 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187578 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017986 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187183 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159914 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200436 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205598 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020497 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352845 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0399356 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179806 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024581 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450287 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0456372 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274756 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183963 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469591 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276004 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207139 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019998 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185065 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194111 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194113 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177527 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214409 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208879 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031038 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243449 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239187 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487917 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289002 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349498 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181873 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186192 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245214 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200921 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199762 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205709 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270721 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257692 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195371 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188834 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170473 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0186217 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168419 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190062 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196096 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195055 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162227 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168748 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149748 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174766 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219544 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170869 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201906 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424437 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022094 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283605 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327944 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184566 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180007 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286034 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023508 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381757 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223485 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283618 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285109 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223756 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373868 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271722 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394067 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352087 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213759 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281963 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309368 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229161 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374993 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239631 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327032 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.044936 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0537223 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445476 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451963 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509744 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0672657 s, 779 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405942 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0696194 s, 753 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682248 s, 768 MB/s 52428800 bytes (52 MB) copied, 0.0484407 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043545 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499882 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265016 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042291 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0333972 s, 1.6 GB/s , 0.0161907 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0185055 s, 2.8 GB/s , 0.0252074 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348325 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215061 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234751 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302214 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236284 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026466 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412884 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251739 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204837 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027134 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218301 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188558 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021282 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160388 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185068 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167435 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235726 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436864 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040079 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434048 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0402417 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486319 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0591469 s, 886 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598875 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704799 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493224 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609697 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549958 s, 953 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510905 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0790859 s, 663 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212574 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0852459 s, 615 MB/s 52428800 bytes (52 MB) copied, 0.0568787 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0849755 s, 617 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392988 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486066 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.076348 s, 687 MB/s 52428800 bytes (52 MB) copied, 0.0432235 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186866 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173295 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021379 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259831 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0414642 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236195 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015413 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160691 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167882 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021854 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170743 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192666 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0182328 s, 2.9 GB/s , 0.0172185 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0160381 s, 3.3 GB/s , 0.0156518 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021026 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163842 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159705 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164493 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201188 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191235 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195845 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168204 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202564 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213581 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203691 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165844 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027483 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175699 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170487 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164649 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231995 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274971 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217526 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383233 s, 1.4 GB/s , 0.0557074 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536231 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320591 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484534 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558878 s, 938 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439525 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586377 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686082 s, 764 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418089 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0488612 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0777565 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0601004 s, 872 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438848 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778135 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681298 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367103 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0711577 s, 737 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843893 s, 621 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504708 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530278 s, 989 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293389 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767317 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664701 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0764124 s, 686 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919416 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249322 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181967 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192815 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022097 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243976 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183451 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184072 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017142 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178306 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253395 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273446 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332651 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355769 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026359 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341438 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441303 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.049758 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355693 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479594 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283731 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0556182 s, 943 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287132 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310796 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303656 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158082 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188535 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329859 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410549 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312689 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297127 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198796 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402909 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556535 s, 942 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0766848 s, 684 MB/s 52428800 bytes (52 MB) copied, 0.0583123 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0895533 s, 585 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402964 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380434 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309673 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270271 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045646 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0190811 s, 2.7 GB/s , 0.0212135 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226042 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204829 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217292 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180732 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186205 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206361 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229666 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271796 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347462 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0227236 s, 2.3 GB/s , 0.0230999 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386811 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326789 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273377 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366931 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315975 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167911 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018025 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322112 s, 1.6 GB/s , 0.0215136 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336244 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287106 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.030948 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449577 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316624 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048338 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.051297 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375127 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0340811 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381402 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401363 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265923 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316827 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199991 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173325 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025231 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174591 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173466 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200631 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261711 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422562 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432699 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0566679 s, 925 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442356 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427082 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0689231 s, 761 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248894 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638423 s, 821 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619504 s, 846 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04397 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056321 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201325 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0634146 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442042 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270142 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295074 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233019 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286693 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265156 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264636 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0175659 s, 3.0 GB/s , 0.019115 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0272906 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255505 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230771 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191284 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379479 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322367 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172516 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240911 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183237 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296036 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160416 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177356 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156124 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157143 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194648 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175697 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176398 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283376 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177398 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186011 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275875 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200878 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316182 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162929 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173482 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220146 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199096 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297513 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205407 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246429 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194055 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220281 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178502 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235006 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279108 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187835 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280433 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258411 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345437 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264578 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294364 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0498011 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210835 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164308 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201941 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207913 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353814 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247493 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279755 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353222 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226945 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184648 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233259 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249566 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186688 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195072 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250742 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389863 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164561 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181283 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200015 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172297 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0253207 s, 2.1 GB/s , 0.0187521 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176934 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171116 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195341 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023792 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198472 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235166 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386589 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264157 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185005 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251458 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348261 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220993 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234213 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287052 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471269 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274284 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036092 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177606 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020768 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186439 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375658 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460918 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352277 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0444379 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602315 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325933 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407781 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477006 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530751 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414586 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406431 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331308 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05298 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0755027 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366562 s, 1.4 GB/s , 0.054765 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232184 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192451 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019318 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191034 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331656 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217001 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207557 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181131 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216744 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268354 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166805 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217576 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358997 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189939 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256492 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534776 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228576 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233517 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250041 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492188 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175896 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465254 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293267 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293375 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226659 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167396 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267574 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016599 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027608 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020984 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309793 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509463 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268574 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514303 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380178 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0685551 s, 765 MB/s 52428800 bytes (52 MB) copied, 0.0787708 s, 666 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626021 s, 837 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805519 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0868752 s, 603 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682394 s, 768 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704585 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0927315 s, 565 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0804127 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700491 s, 748 MB/s 52428800 bytes (52 MB) copied, 0.0394419 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0994385 s, 527 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373883 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103932 s, 504 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116405 s, 450 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811296 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430464 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285645 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422445 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510444 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316904 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165258 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160549 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197097 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231934 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206943 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176951 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196912 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289004 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195347 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414324 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039367 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0299071 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173619 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242595 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488687 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025206 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218583 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0417347 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476479 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285686 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599081 s, 875 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463285 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264519 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332693 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344371 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321652 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320214 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187887 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197734 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181249 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170372 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170427 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017055 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171089 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172311 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01814 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169957 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289515 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190477 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330421 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321581 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224141 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251002 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539789 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172751 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022148 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194707 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023571 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252144 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018863 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215347 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0251699 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249274 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338542 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293126 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040465 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281485 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380946 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03684 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374751 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209947 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359014 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402866 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515895 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398681 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414534 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183765 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390401 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269783 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207685 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020698 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271355 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202143 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216997 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424162 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372511 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426855 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399618 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362183 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421143 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036231 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042674 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352755 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426259 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0417427 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220889 s, 2.4 GB/s , 0.0385186 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326038 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352478 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300048 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159748 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358322 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020989 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199527 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226438 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269685 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160923 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169249 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218266 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328587 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194149 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363842 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213261 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363349 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170907 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204525 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229692 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177364 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222037 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171237 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023776 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216096 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214801 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249385 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253886 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230633 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241961 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256536 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198088 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0372143 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0325315 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302464 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333404 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198783 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427882 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255392 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407577 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367258 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357093 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320315 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0503471 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464948 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613653 s, 854 MB/s 52428800 bytes (52 MB) copied, 0.044559 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414953 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245536 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285551 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325762 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195292 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347042 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229013 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01967 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262312 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268222 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215766 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188637 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300709 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322076 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016053 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174563 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183766 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230677 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270284 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228069 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301261 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307174 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178443 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234166 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160421 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186063 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224211 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191511 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208612 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0619696 s, 846 MB/s 52428800 bytes (52 MB) copied, 0.0546258 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383967 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686556 s, 764 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249567 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467035 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639774 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504107 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0594356 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210794 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438535 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393081 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250282 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310713 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404623 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.050854 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381871 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330919 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426032 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435858 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337916 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199386 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244163 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176284 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0843383 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462299 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341825 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204659 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181864 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284368 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386731 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183923 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277914 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298188 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501356 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376953 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248011 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211393 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450605 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219215 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489418 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233919 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163879 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241779 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259219 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399794 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492444 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0384664 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0223051 s, 2.4 GB/s , 0.0444649 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384912 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182381 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018808 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302998 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302367 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173634 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300678 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421402 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190896 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246867 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028242 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232245 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268832 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200349 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0254624 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198617 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253814 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221937 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02853 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040158 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421334 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243191 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583349 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372802 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319783 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410873 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036925 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340036 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.056096 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538633 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178401 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438605 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223246 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538754 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037508 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198478 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0371847 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209095 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353307 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236785 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373089 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245399 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180194 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414406 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372214 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0487577 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479883 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438095 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305046 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371705 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177168 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390201 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400661 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313768 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322295 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354658 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231329 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197334 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227018 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0333718 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252106 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298588 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195712 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436707 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360265 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0570144 s, 920 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0718995 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123996 s, 423 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0638544 s, 821 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0800638 s, 655 MB/s 52428800 bytes (52 MB) copied, 0.093648 s, 560 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0811135 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0826419 s, 634 MB/s 52428800 bytes (52 MB) copied, 0.0911744 s, 575 MB/s 52428800 bytes (52 MB) copied, 0.100002 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809073 s, 648 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0864865 s, 606 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875656 s, 599 MB/s 52428800 bytes (52 MB) copied, 0.0647679 s, 809 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882802 s, 594 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136956 s, 383 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0772545 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.074387 s, 705 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447853 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0526288 s, 996 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395121 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364903 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276576 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195117 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193823 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306881 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250443 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512205 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336562 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286201 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209734 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609153 s, 861 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401767 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358663 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204278 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231794 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390909 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347041 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372193 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251508 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282422 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027453 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054913 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270868 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214794 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038717 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214773 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.034192 s, 1.5 GB/s , 0.0193681 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385125 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236193 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294981 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296993 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511028 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288853 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043277 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247822 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164308 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221944 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314796 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030113 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211571 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258331 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292005 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024172 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359642 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303917 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229946 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215231 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185558 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267529 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431676 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397047 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0499431 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368918 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310958 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022452 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269011 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460831 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0660052 s, 794 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024067 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02838 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047599 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030831 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297619 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323025 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193704 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365197 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287111 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063447 s, 826 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0370544 s, 1.4 GB/s , 0.0489208 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692543 s, 757 MB/s 52428800 bytes (52 MB) copied, 0.0316081 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302834 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176189 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438968 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226279 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467064 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0352778 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633385 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273587 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379395 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296175 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159652 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285267 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236614 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953027 s, 550 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060665 s, 864 MB/s 52428800 bytes (52 MB) copied, 0.034463 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0698884 s, 750 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413253 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0351868 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066627 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419049 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387479 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065718 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124191 s, 422 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106239 s, 493 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235787 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439161 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234907 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225334 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222401 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327033 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358307 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296801 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327469 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0313651 s, 1.7 GB/s , 0.0353254 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287405 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416829 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0430242 s, 1.2 GB/s , 0.0397683 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369054 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0476747 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519676 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306998 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264694 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883473 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0875944 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457353 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530953 s, 987 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429997 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442941 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0377383 s, 1.4 GB/s , 0.0429666 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314935 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274044 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523315 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207723 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373646 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0552091 s, 950 MB/s 52428800 bytes (52 MB) copied, 0.039352 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163818 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464551 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294423 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505741 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522676 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0892052 s, 588 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425554 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.04202 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030851 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0853342 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627313 s, 836 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791059 s, 663 MB/s 52428800 bytes (52 MB) copied, 0.0454205 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429659 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418939 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423804 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360947 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408765 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032789 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200894 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158187 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165949 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188639 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029436 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326251 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189877 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020086 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338967 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022655 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0542043 s, 967 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194036 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532029 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489302 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404842 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043126 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236078 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.060268 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186104 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368826 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300493 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0507173 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267705 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368527 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390481 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442377 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276125 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363271 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0683191 s, 767 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300711 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299135 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309236 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379348 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179577 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391641 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028342 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031718 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343693 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241319 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185771 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308041 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030084 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240448 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194046 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434898 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259658 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445861 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337941 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327776 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.092432 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373742 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405397 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299505 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628294 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541139 s, 969 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486593 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381615 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342349 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432312 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358345 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369857 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218792 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310546 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018245 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0742373 s, 706 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325931 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195713 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0626806 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324548 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432238 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715416 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354415 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558525 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202932 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457185 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642913 s, 815 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923033 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0670697 s, 782 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104011 s, 504 MB/s 52428800 bytes (52 MB) copied, 0.0693801 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0965164 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0964818 s, 543 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.155475 s, 337 MB/s 52428800 bytes (52 MB) copied, 0.110763 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106954 s, 490 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0955774 s, 549 MB/s 52428800 bytes (52 MB) copied, 0.0997612 s, 526 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.102862 s, 510 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0953918 s, 550 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108186 s, 485 MB/s 52428800 bytes (52 MB) copied, 0.110276 s, 475 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.144873 s, 362 MB/s 52428800 bytes (52 MB) copied, 0.180629 s, 290 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112668 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0876399 s, 598 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.164103 s, 319 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248092 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130927 s, 400 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.094436 s, 555 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122974 s, 426 MB/s 52428800 bytes (52 MB) copied, 0.0773635 s, 678 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0806105 s, 650 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0961525 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0841368 s, 623 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0883845 s, 593 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110574 s, 474 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0572519 s, 916 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110362 s, 475 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0571488 s, 917 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0513567 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474963 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322512 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269178 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264062 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180515 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175582 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210882 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016631 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365915 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054381 s, 964 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424695 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514999 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666574 s, 787 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767772 s, 683 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728606 s, 720 MB/s 52428800 bytes (52 MB) copied, 0.0960661 s, 546 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0816484 s, 642 MB/s 52428800 bytes (52 MB) copied, 0.135446 s, 387 MB/s 52428800 bytes (52 MB) copied, 0.113725 s, 461 MB/s 52428800 bytes (52 MB) copied, 0.117489 s, 446 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0600436 s, 873 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.137759 s, 381 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112914 s, 464 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0941569 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128544 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117031 s, 448 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0641136 s, 818 MB/s , 0.0610392 s, 859 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0468357 s, 1.1 GB/s , 0.0491056 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109899 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0680589 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304887 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275569 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178342 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231853 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378875 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291253 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300615 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207463 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286901 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191378 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034782 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202893 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0639477 s, 820 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315143 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028136 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317794 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038813 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284925 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0505434 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0842971 s, 622 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0710675 s, 738 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401872 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0613258 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297698 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428891 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0977239 s, 536 MB/s , 0.0616547 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246989 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317713 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0826695 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557332 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578021 s, 907 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502366 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362883 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0389794 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.108095 s, 485 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114852 s, 456 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516125 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0613567 s, 854 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461026 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769941 s, 681 MB/s 52428800 bytes (52 MB) copied, 0.0480345 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673976 s, 778 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405927 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.12042 s, 435 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440177 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116573 s, 450 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113652 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.106851 s, 491 MB/s 52428800 bytes (52 MB) copied, 0.132341 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421594 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0734175 s, 714 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292908 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211907 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166288 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354453 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027502 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328976 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367039 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0673101 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662663 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0750827 s, 698 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473592 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378918 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0442827 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271114 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297314 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0510306 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259362 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420932 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417261 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252827 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198525 s, 2.6 GB/s , 0.0312111 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177346 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175543 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396994 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278305 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219942 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219641 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285335 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222036 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016455 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233016 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367757 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178558 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484548 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183104 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544826 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284937 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189783 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246821 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298614 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781237 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482794 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032875 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039643 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614582 s, 853 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058913 s, 890 MB/s 52428800 bytes (52 MB) copied, 0.0635052 s, 826 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609405 s, 860 MB/s 52428800 bytes (52 MB) copied, 0.090839 s, 577 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0855854 s, 613 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0993794 s, 528 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0795628 s, 659 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108764 s, 482 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119122 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162365 s, 323 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.103285 s, 508 MB/s , 0.132053 s, 397 MB/s 52428800 bytes (52 MB) copied, 0.168787 s, 311 MB/s 52428800 bytes (52 MB) copied, 0.126157 s, 416 MB/s 52428800 bytes (52 MB) copied, 0.138029 s, 380 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11178 s, 469 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.084842 s, 618 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180303 s, 291 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736714 s, 712 MB/s 52428800 bytes (52 MB) copied, 0.0557197 s, 941 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159062 s, 330 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0839769 s, 624 MB/s 52428800 bytes (52 MB) copied, 0.0741493 s, 707 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0632277 s, 829 MB/s 52428800 bytes (52 MB) copied, 0.128464 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.124586 s, 421 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516797 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341925 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119878 s, 437 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625977 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716967 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345511 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220794 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276349 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178401 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180717 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300105 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0430361 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351012 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294383 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408028 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335746 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281703 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273669 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215906 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319017 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177866 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248893 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333302 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458764 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0704878 s, 744 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246891 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653022 s, 803 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0682688 s, 768 MB/s , 0.0673021 s, 779 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662683 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663814 s, 790 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579347 s, 905 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791712 s, 662 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726195 s, 722 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0791773 s, 662 MB/s 52428800 bytes (52 MB) copied, 0.0904015 s, 580 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0884588 s, 593 MB/s 52428800 bytes (52 MB) copied, 0.0752204 s, 697 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.134253 s, 391 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141655 s, 370 MB/s 52428800 bytes (52 MB) copied, 0.143237 s, 366 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0691035 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827631 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064496 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0870702 s, 602 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0776578 s, 675 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452824 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514387 s, 1.0 GB/s , 0.0887305 s, 591 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.016103 s, 3.3 GB/s 52428800 bytes (52 MB) copied, 0.0443692 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289437 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655454 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188397 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180031 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180862 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204947 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022373 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290182 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301932 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168714 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352357 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025418 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534008 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299496 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216573 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286268 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274033 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474325 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421495 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0528741 s, 992 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779144 s, 673 MB/s 52428800 bytes (52 MB) copied, 0.10321 s, 508 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0854447 s, 614 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112015 s, 468 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721854 s, 726 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0654198 s, 801 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0966667 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135977 s, 386 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.159925 s, 328 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141858 s, 370 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158978 s, 330 MB/s 52428800 bytes (52 MB) copied, 0.0940498 s, 557 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.147743 s, 355 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107946 s, 486 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0946582 s, 554 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146361 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0618695 s, 847 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454959 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379828 s, 1.4 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.140031 s, 374 MB/s 52428800 bytes (52 MB) copied, 0.118053 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412096 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130656 s, 401 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0927766 s, 565 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266643 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.125401 s, 418 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695259 s, 754 MB/s 52428800 bytes (52 MB) copied, 0.146441 s, 358 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238907 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769179 s, 682 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228905 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180868 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159431 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310147 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392844 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424392 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291285 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243767 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223158 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447621 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0525428 s, 998 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521033 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307773 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169814 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335928 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224707 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439392 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0656915 s, 798 MB/s 52428800 bytes (52 MB) copied, 0.0238538 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561124 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035552 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109 s, 481 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771545 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336256 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0820734 s, 639 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0637771 s, 822 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507117 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260023 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115219 s, 455 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113431 s, 462 MB/s , 0.0759953 s, 690 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0918641 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583079 s, 899 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0539587 s, 972 MB/s , 0.110848 s, 473 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104665 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0741624 s, 707 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276043 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576211 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295488 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557844 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0808035 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232052 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0789771 s, 664 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103024 s, 509 MB/s 52428800 bytes (52 MB) copied, 0.0526056 s, 997 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450502 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0653119 s, 803 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247785 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281572 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551102 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.117738 s, 445 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202321 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226333 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350651 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022229 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380114 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0592829 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0759967 s, 690 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377998 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0567202 s, 924 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436552 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0769467 s, 681 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057748 s, 908 MB/s 52428800 bytes (52 MB) copied, 0.0594522 s, 882 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0871647 s, 601 MB/s 52428800 bytes (52 MB) copied, 0.0827711 s, 633 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0754958 s, 694 MB/s 52428800 bytes (52 MB) copied, 0.0590555 s, 888 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419537 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0502025 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423825 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398234 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482435 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.119099 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430967 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549018 s, 955 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0962552 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0384146 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480668 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556577 s, 942 MB/s 52428800 bytes (52 MB) copied, 0.0700524 s, 748 MB/s , 0.0391854 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380649 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358841 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.081921 s, 640 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357123 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247603 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349285 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426706 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0950748 s, 551 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0943055 s, 556 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586236 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020535 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225323 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558138 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501341 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188247 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212329 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265237 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364778 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561061 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504014 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419806 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297987 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0666824 s, 786 MB/s 52428800 bytes (52 MB) copied, 0.0939134 s, 558 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.075563 s, 694 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.115936 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0830715 s, 631 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0735031 s, 713 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879467 s, 596 MB/s 52428800 bytes (52 MB) copied, 0.0793014 s, 661 MB/s 52428800 bytes (52 MB) copied, 0.143023 s, 367 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0723316 s, 725 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107486 s, 488 MB/s 52428800 bytes (52 MB) copied, 0.115615 s, 453 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0753672 s, 696 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.057038 s, 919 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113439 s, 462 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0752445 s, 697 MB/s , 0.0718995 s, 729 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0721484 s, 727 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505047 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0823486 s, 637 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0728418 s, 720 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468417 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271598 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194476 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0634108 s, 827 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038834 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201646 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209585 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368481 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029159 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018706 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0298463 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221424 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283305 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290771 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020403 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275849 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191272 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200164 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01862 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296046 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284294 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401041 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266072 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419625 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040494 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035443 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290191 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576395 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032511 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258408 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408877 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340214 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0586556 s, 894 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425256 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260949 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330268 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359453 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0545177 s, 962 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385172 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330142 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.081495 s, 643 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544092 s, 964 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0726127 s, 722 MB/s 52428800 bytes (52 MB) copied, 0.0602045 s, 871 MB/s 52428800 bytes (52 MB) copied, 0.042863 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069106 s, 759 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642353 s, 816 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11374 s, 461 MB/s 52428800 bytes (52 MB) copied, 0.0906627 s, 578 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0633466 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644924 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0606601 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0551488 s, 951 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557464 s, 940 MB/s 52428800 bytes (52 MB) copied, 0.0623054 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0692489 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493749 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0537054 s, 976 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189751 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205094 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204446 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016539 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434822 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299811 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230628 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463353 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328393 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623012 s, 842 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285469 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026114 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0523705 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372836 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320266 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267997 s, 2.0 GB/s 52428800 bytes (52 MB) copied, 0.0454837 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437332 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318002 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488606 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327981 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340347 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246389 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247201 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578269 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435354 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182499 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194986 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198386 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210245 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205228 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175245 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184266 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316049 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0500636 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212761 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271925 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047173 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286958 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022258 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348031 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322252 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198626 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184461 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0344263 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181072 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293205 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576811 s, 909 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288402 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041496 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331467 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534062 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505564 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207943 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505712 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470876 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264988 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187812 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187429 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276856 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314439 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449859 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539729 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254699 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041179 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393306 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295815 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227388 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234344 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175373 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321078 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290363 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0417319 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482946 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296622 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197431 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038292 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468999 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197647 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458365 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039264 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339107 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390904 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397413 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365332 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640378 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564798 s, 928 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360566 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0432209 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364465 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227595 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042027 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519204 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0603775 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226984 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208717 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157946 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164564 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208093 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251198 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188974 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166969 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516007 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404809 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379314 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336859 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199463 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152448 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218427 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163521 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265093 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196473 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203215 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216626 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324111 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186734 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161889 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177061 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165185 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200611 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208475 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187031 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177616 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173874 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175043 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236688 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183964 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174843 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018656 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017133 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018764 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390153 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269841 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357137 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336374 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257358 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233283 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368561 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484919 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225109 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187624 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296524 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166981 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223998 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151913 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176157 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151082 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296868 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209516 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204449 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223584 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194709 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225915 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267553 s, 2.0 GB/s , 0.0298228 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181551 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02045 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169497 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221669 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319401 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258239 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219662 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190543 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220149 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228759 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284901 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392691 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0548732 s, 955 MB/s 52428800 bytes (52 MB) copied, 0.0459015 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0517548 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0488165 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540317 s, 970 MB/s 52428800 bytes (52 MB) copied, 0.0626948 s, 836 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496875 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0485599 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439021 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290767 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490623 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184926 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221331 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184438 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.018148 s, 2.9 GB/s , 0.017853 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160481 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210352 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181578 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202393 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166951 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183606 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227753 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175414 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168123 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229481 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229277 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165606 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226216 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177691 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0389784 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221651 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499835 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0516778 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440444 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0426363 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496861 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.036946 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357742 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585069 s, 896 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198535 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341513 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281227 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625181 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321576 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276607 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230731 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280685 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021905 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248183 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265902 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225602 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282674 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441748 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0271619 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213994 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0374687 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312466 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357798 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278167 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349766 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.098243 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603687 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298462 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230088 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025064 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230533 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252637 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285959 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225088 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247341 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268871 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241179 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224533 s, 2.3 GB/s , 0.0221922 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198584 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.02189 s, 2.4 GB/s , 0.0254244 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260196 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204485 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217269 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207842 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233324 s, 2.2 GB/s , 0.0187798 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194083 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187755 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165596 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172494 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020529 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181476 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015542 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220449 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213568 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214169 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173225 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021052 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379966 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416282 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357755 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0352219 s, 1.5 GB/s , 0.0399693 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210758 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399681 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037215 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206726 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176069 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196965 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197919 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214423 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247872 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261939 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256066 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182479 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225143 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0419809 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184383 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471047 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020697 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327961 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241027 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0278064 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237034 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191929 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205339 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273899 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374525 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180957 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230251 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321165 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222273 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180296 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356243 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037407 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263774 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509609 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401516 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298275 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334237 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0450395 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239506 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0645203 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461988 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200962 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408976 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564994 s, 928 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212485 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383307 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243565 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470959 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318028 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361661 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260061 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429871 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332016 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0276018 s, 1.9 GB/s , 0.0231453 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237948 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347115 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031859 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221015 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580729 s, 903 MB/s 52428800 bytes (52 MB) copied, 0.0285253 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397069 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267783 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269481 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0524395 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395248 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304041 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236776 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186857 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019292 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205188 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275732 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187608 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208509 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027286 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180876 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299017 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199389 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298028 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0231183 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213446 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230373 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185664 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179598 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171368 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172228 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198002 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257664 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224183 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252249 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213524 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020888 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201918 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167458 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018737 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194961 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181123 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212606 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.041488 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285539 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419289 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171635 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040107 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326813 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025831 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342524 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383339 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029456 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0325843 s, 1.6 GB/s , 0.0500273 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264974 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220963 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220278 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568939 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181527 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540598 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022404 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0178261 s, 2.9 GB/s , 0.0223483 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152067 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180747 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154573 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194731 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250519 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021131 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208284 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016495 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165551 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213791 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283713 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255379 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349239 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03818 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397515 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039392 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0782742 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440701 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0625738 s, 838 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416835 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482386 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0375251 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493927 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0627959 s, 835 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380447 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0663154 s, 791 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274566 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01625 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246159 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325339 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194953 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191828 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186878 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151766 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160565 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024283 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193579 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193693 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287462 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256162 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301487 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428631 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033522 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355381 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218643 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236388 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492588 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183206 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0314203 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232339 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170951 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251716 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165464 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188936 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250429 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202069 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302114 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198503 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186817 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189722 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016249 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365272 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205638 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195538 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251417 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168779 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205492 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176997 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208472 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213495 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215175 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209807 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291237 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213704 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177421 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251774 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245138 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243828 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224369 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187458 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171201 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185073 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018864 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217367 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175706 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166815 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186073 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319121 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021306 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193526 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183083 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164915 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195066 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193549 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287765 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229643 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317617 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250377 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225706 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283266 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173301 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192536 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163523 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016452 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190977 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022829 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163583 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182396 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181499 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293654 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381259 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.039218 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198213 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302567 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219928 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407652 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202896 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193163 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198841 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257419 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239236 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290435 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311746 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223418 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243362 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225036 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237365 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037981 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185538 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285124 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184227 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196556 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283827 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194892 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299323 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0147137 s, 3.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208314 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164145 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205193 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187793 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201738 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221673 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029777 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030191 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177297 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454164 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232239 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168276 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184409 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295584 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180349 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019439 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250204 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162562 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181655 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186323 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017839 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240888 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309081 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346608 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342385 s, 1.5 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298236 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185381 s, 2.8 GB/s , 0.058105 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166873 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423436 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333072 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.065175 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0490077 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346313 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0464151 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0459642 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0608381 s, 862 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.066986 s, 783 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031314 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0447783 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264603 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331514 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306406 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512483 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345606 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248457 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196094 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259704 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297596 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236234 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166046 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235434 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177069 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168878 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452659 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315742 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257137 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492891 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02082 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317083 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386765 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220989 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451417 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0343 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205569 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035609 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466764 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329403 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414205 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199538 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207945 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336047 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282354 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0387025 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017708 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381484 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237613 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242321 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727521 s, 721 MB/s 52428800 bytes (52 MB) copied, 0.0339831 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609915 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623115 s, 841 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339169 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.055095 s, 952 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0530515 s, 988 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307357 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0828015 s, 633 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422687 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476666 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040542 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217626 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243912 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473015 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216929 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0386968 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237153 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189632 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164198 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016969 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285027 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239613 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249554 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374104 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033074 s, 1.6 GB/s , 0.0223072 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321073 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181053 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269093 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433687 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275776 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436011 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366734 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247411 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186583 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174022 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204103 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240835 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157305 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239625 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016956 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257765 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255468 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194058 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208191 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231487 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230073 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366642 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418201 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297286 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363531 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335403 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257457 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432258 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412202 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452544 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369425 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.043158 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.05161 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294381 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356575 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0714326 s, 734 MB/s 52428800 bytes (52 MB) copied, 0.0495773 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457602 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0560299 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040827 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716992 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0900754 s, 582 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0623945 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494715 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402617 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584291 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179704 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211365 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472706 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237372 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183607 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182078 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020365 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263316 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181936 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177354 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195049 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324513 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322208 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0192498 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155138 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241585 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162999 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205976 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175965 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205996 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202036 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038641 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415757 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318162 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186139 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372292 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215059 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210488 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199502 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457558 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187695 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311936 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196467 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193992 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207336 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188434 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230868 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0186192 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0200034 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175713 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312408 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200687 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169096 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022431 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169293 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022139 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233753 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225605 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311679 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048614 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261546 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194903 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040845 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0357549 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333636 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.03165 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273411 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283574 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187266 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405623 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0193898 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226842 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185513 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186621 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236292 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02651 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020847 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182228 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017857 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219815 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181178 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180917 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180744 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197897 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361373 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225414 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275421 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0445937 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655554 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371528 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0364472 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392664 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0306932 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644428 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410645 s, 1.3 GB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0452263 s, 1.2 GB/s , 0.0484221 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0539179 s, 972 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0507916 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0659415 s, 795 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612021 s, 857 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499664 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.099052 s, 529 MB/s 52428800 bytes (52 MB) copied, 0.0919601 s, 570 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716751 s, 731 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520247 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737426 s, 711 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0441719 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0555045 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0584295 s, 897 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0568844 s, 922 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237745 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113987 s, 460 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519832 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133799 s, 392 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353329 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.093511 s, 561 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512392 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0535948 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423879 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242238 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0418543 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.069292 s, 757 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536172 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190555 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0850825 s, 616 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326851 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313743 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457367 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291645 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0807237 s, 649 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240153 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284834 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0582912 s, 899 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0664803 s, 789 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250754 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195089 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0558266 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262107 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192476 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451906 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173779 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197631 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022743 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195866 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223047 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171723 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342853 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339633 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203837 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231305 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190183 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356054 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168905 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204721 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302603 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025486 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268343 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401758 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509171 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203736 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380171 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351295 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227212 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017271 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220839 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170837 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200822 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016951 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170414 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227295 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191796 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285294 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199826 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242337 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241722 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173512 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268334 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198314 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181713 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197727 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0191171 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016126 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177224 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02324 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027789 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209578 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198361 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029511 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318579 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223697 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173753 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215946 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174135 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018916 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309895 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236767 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240806 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199005 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200068 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261336 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202611 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017268 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205513 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315354 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173653 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182087 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251875 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187252 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378674 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283529 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0325483 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273985 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0602471 s, 870 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252751 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274252 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028215 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0651001 s, 805 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477642 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472159 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560075 s, 936 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259448 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242572 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318935 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261486 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201658 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197366 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438643 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039056 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255064 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019465 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286192 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250045 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256367 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0515249 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521018 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347698 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326035 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180601 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183685 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177112 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161891 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179556 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228735 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172409 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206188 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210147 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171553 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198552 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016016 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188595 s, 2.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318187 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0201909 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174087 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235437 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02011 s, 2.6 GB/s 52428800 bytes (52 MB) copied, 0.0214874 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274792 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211138 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190111 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310916 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194085 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272444 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169406 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211968 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294449 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174593 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264032 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197651 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178323 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163819 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177567 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208472 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242627 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312023 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170074 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172775 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169158 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173863 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016136 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196004 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277712 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018667 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218211 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0395844 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273252 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313853 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0192487 s, 2.7 GB/s 52428800 bytes (52 MB) copied, 0.0283339 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501511 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025749 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177661 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343593 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190538 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166383 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191284 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257624 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303208 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231288 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0399831 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426617 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438579 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034992 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433226 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406283 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391657 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392622 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047263 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383065 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0409591 s, 1.3 GB/s , 0.0424094 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01702 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316816 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253717 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016253 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282172 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404746 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216771 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339773 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298385 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330169 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284941 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253465 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296344 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016367 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031761 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320315 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247491 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0467737 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408457 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296135 s, 1.8 GB/s , 0.0545699 s, 961 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0960266 s, 546 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580624 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509018 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0919722 s, 570 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0884759 s, 593 MB/s 52428800 bytes (52 MB) copied, 0.09443 s, 555 MB/s 52428800 bytes (52 MB) copied, 0.0984994 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0763253 s, 687 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0979459 s, 535 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.107429 s, 488 MB/s 50+0 records in 50+0 records out , 0.0778013 s, 674 MB/s 52428800 bytes (52 MB) copied, 0.0983023 s, 533 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103813 s, 505 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100852 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.122346 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0885795 s, 592 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598615 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112472 s, 466 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0560961 s, 935 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0770969 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342228 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.123128 s, 426 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695286 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.062916 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240008 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644417 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378109 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280731 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189897 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0767779 s, 683 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344757 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0538903 s, 973 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183424 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178471 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190066 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01904 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167021 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201389 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288601 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263093 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188709 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175158 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211237 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177807 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289926 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362263 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320949 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468141 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275372 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0416911 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346907 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362131 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0535987 s, 978 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317722 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169472 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655283 s, 800 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194958 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354019 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0621027 s, 844 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300973 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151843 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259647 s, 2.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296813 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0446309 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161108 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017543 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293869 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343398 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466681 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174337 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0693757 s, 756 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342883 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424244 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0737661 s, 711 MB/s 52428800 bytes (52 MB) copied, 0.0361376 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0488874 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0702697 s, 746 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0657243 s, 798 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0549261 s, 955 MB/s 52428800 bytes (52 MB) copied, 0.0834334 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0505691 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0458477 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466282 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039184 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022737 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394924 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247329 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0546177 s, 960 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433803 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216149 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0478698 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562697 s, 932 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368503 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0433759 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0464387 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182002 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171616 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404379 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242167 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371033 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235281 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246981 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218991 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021077 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263354 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270438 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494887 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333971 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177157 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319943 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331067 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169357 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398443 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0440215 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305031 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337175 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037328 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0691454 s, 758 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354012 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371424 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0548243 s, 956 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0727039 s, 721 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031045 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0604967 s, 867 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607401 s, 863 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421558 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463414 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812033 s, 646 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063277 s, 829 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278531 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640808 s, 818 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0405392 s, 1.3 GB/s 52428800 bytes (52 MB) copied, 0.0522358 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0660981 s, 793 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270166 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0536669 s, 977 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198746 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359558 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205521 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175797 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241598 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235014 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256063 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254065 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016537 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218578 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210937 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164116 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290312 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377553 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280655 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176316 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0479592 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297845 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168198 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248351 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331154 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212619 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218336 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425321 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0424313 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0380909 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048238 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0695503 s, 754 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0882477 s, 594 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.121523 s, 431 MB/s 52428800 bytes (52 MB) copied, 0.104086 s, 504 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112456 s, 466 MB/s 52428800 bytes (52 MB) copied, 0.0879388 s, 596 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118151 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100026 s, 524 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.1221 s, 429 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110933 s, 473 MB/s 52428800 bytes (52 MB) copied, 0.100791 s, 520 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118669 s, 442 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107633 s, 487 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832476 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110777 s, 473 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105944 s, 495 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.139702 s, 375 MB/s , 0.0838988 s, 625 MB/s 52428800 bytes (52 MB) copied, 0.0917444 s, 571 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13144 s, 399 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109025 s, 481 MB/s 52428800 bytes (52 MB) copied, 0.120951 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.101208 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112511 s, 466 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273735 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277241 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199441 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018391 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214063 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028886 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185628 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216875 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354818 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205118 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468844 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332512 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350133 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335814 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194099 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260098 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273481 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367267 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0515518 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382618 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500049 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0662355 s, 792 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731109 s, 717 MB/s 52428800 bytes (52 MB) copied, 0.0535187 s, 980 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296675 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0594162 s, 882 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0701693 s, 747 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315568 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628916 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024518 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235195 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168152 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194527 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278973 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430183 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304937 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209828 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321379 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248951 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0473414 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0344371 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585405 s, 896 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0805299 s, 651 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0603969 s, 868 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0597777 s, 877 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0708484 s, 740 MB/s 52428800 bytes (52 MB) copied, 0.0782569 s, 670 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04584 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221206 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0440024 s, 1.2 GB/s , 0.0577437 s, 908 MB/s 52428800 bytes (52 MB) copied, 0.0891853 s, 588 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486826 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596849 s, 878 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430255 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217239 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183279 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170871 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198669 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179122 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194749 s, 2.7 GB/s , 0.0222739 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216896 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190502 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214414 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198775 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320821 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295204 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292669 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271613 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269168 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342301 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366795 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235798 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236938 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244038 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219531 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192386 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170741 s, 3.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0184498 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0270621 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0199966 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184746 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224325 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205645 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468201 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0470186 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036144 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425831 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501785 s, 1.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351631 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377401 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0407916 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.064433 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265905 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463219 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540583 s, 970 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185008 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180326 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287096 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446458 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168714 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266778 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203484 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276321 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211204 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341447 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190802 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213231 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390432 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031486 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271255 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0394392 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261383 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0477769 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016814 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317857 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194095 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227143 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214279 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162435 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217603 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016286 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189882 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0284562 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374975 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0373445 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281256 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322529 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367432 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0523687 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367136 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262701 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0504609 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302931 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197915 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174128 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0227958 s, 2.3 GB/s , 0.0228374 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172566 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211194 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0494486 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329569 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0446598 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472762 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457427 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306626 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0720149 s, 728 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240827 s, 2.2 GB/s 52428800 bytes (52 MB) copied, 0.0588936 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415139 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0640012 s, 819 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379173 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0461164 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235871 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196793 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168251 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250502 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396188 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0311296 s, 1.7 GB/s , 0.0181718 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276762 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184182 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211863 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336072 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186842 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289259 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305443 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277829 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329448 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274149 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309688 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342066 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356274 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349426 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263141 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334771 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0453432 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779315 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554617 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397462 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0520883 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484779 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0588963 s, 890 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681682 s, 769 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0414507 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0677243 s, 774 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644492 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.056214 s, 933 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0612616 s, 856 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365886 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193567 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361947 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519934 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359962 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361676 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160723 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210651 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026947 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356438 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169728 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351332 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325308 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255424 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355948 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0730255 s, 718 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217741 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02713 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174675 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236354 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267372 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213104 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216499 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180358 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351419 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197063 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234979 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229613 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194124 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398413 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235978 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190482 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277313 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229773 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242034 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443295 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472932 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204485 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242506 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016535 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312658 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172393 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0472322 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186574 s, 2.8 GB/s , 0.0299859 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335164 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029481 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166891 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403543 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280924 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256756 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0482907 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330385 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0396259 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180224 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466166 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335853 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321549 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262873 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252711 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246512 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0534026 s, 982 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0443625 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333885 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040228 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187622 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319413 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0366143 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239232 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255865 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030785 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264806 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230599 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275016 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0382004 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295115 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231076 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189611 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269012 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191804 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214005 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291614 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313254 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222897 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481287 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476866 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.037086 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.029852 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0500034 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377378 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.048249 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0610693 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0581072 s, 902 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0512117 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0541629 s, 968 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595825 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.08269 s, 634 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287285 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286373 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771684 s, 679 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290978 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415625 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179978 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265765 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308676 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336294 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040318 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298921 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328246 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0413134 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0468201 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202302 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0575471 s, 911 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0529655 s, 990 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0404553 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644333 s, 814 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484477 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0700655 s, 748 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0628621 s, 834 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835028 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0595814 s, 880 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0573306 s, 914 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0879684 s, 596 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118376 s, 443 MB/s 52428800 bytes (52 MB) copied, 0.0957045 s, 548 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0644999 s, 813 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0649281 s, 807 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0620275 s, 845 MB/s 52428800 bytes (52 MB) copied, 0.140998 s, 372 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931722 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496819 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0681271 s, 770 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153361 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0607146 s, 864 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.128366 s, 408 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0686856 s, 763 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100115 s, 524 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814153 s, 644 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658747 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348148 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152909 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0474071 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411104 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379297 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0305025 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280752 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206102 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275875 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025299 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160867 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263062 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296299 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162451 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299048 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360069 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196157 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317601 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302192 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209951 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269547 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306449 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277467 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.033801 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286255 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0492376 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0438714 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0514194 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045174 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350519 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0363766 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421631 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0579733 s, 904 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213634 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658481 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0609538 s, 860 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305261 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345336 s, 1.5 GB/s , 0.0587464 s, 892 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967304 s, 542 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0376022 s, 1.4 GB/s , 0.0563209 s, 931 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243942 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037325 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354386 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0372515 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197539 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202582 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285478 s, 1.8 GB/s , 0.0303947 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040784 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419842 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0532526 s, 985 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029838 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039524 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0542624 s, 966 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067248 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167538 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340675 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564487 s, 929 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0948568 s, 553 MB/s 52428800 bytes (52 MB) copied, 0.0715819 s, 732 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0712726 s, 736 MB/s 52428800 bytes (52 MB) copied, 0.152772 s, 343 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0975983 s, 537 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0432533 s, 1.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0732186 s, 716 MB/s 52428800 bytes (52 MB) copied, 0.0643759 s, 814 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0599518 s, 875 MB/s 52428800 bytes (52 MB) copied, 0.0398895 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225209 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116885 s, 449 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158544 s, 331 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0974465 s, 538 MB/s , 0.0922594 s, 568 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.100611 s, 521 MB/s 52428800 bytes (52 MB) copied, 0.160207 s, 327 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0847312 s, 619 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239065 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.243106 s, 216 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248595 s, 211 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150606 s, 348 MB/s 52428800 bytes (52 MB) copied, 0.198689 s, 264 MB/s 52428800 bytes (52 MB) copied, 0.118762 s, 441 MB/s 52428800 bytes (52 MB) copied, 0.116671 s, 449 MB/s 52428800 bytes (52 MB) copied, 0.232485 s, 226 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211879 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.211532 s, 248 MB/s 52428800 bytes (52 MB) copied, 0.210064 s, 250 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.141195 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.154639 s, 339 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214978 s, 244 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162638 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0845764 s, 620 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120691 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107476 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0758215 s, 691 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.180297 s, 291 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118394 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377656 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0931502 s, 563 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112827 s, 465 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0615771 s, 851 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0835466 s, 628 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059329 s, 884 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0547918 s, 957 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0428009 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247407 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310385 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0614617 s, 853 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465386 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373248 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379169 s, 1.4 GB/s 52428800 bytes (52 MB) copied, 0.0248691 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035627 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022674 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042372 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199931 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0803991 s, 652 MB/s 52428800 bytes (52 MB) copied, 0.0599841 s, 874 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501548 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168609 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254485 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017006 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308227 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170767 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212066 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319815 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287025 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434106 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429973 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324921 s, 1.6 GB/s 52428800 bytes (52 MB) copied, 0.0523017 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211754 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320742 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491456 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0367305 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030982 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355976 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0556074 s, 943 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0385931 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326118 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569338 s, 921 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420095 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.11388 s, 460 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167735 s, 3.1 GB/s 52428800 bytes (52 MB) copied, 0.0249603 s, 2.1 GB/s 52428800 bytes (52 MB) copied, 0.0519638 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402913 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259037 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392319 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252699 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0642386 s, 816 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035335 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0475971 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181981 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0558593 s, 939 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172912 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0580671 s, 903 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019085 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226716 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218998 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292198 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162807 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355237 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256111 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190945 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178409 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035577 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0291268 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266637 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190095 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220854 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320989 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218899 s, 2.4 GB/s 52428800 bytes (52 MB) copied, 0.0524431 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204352 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0282103 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302846 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167641 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179864 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015512 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028073 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271098 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225499 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0797378 s, 658 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0489341 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0633399 s, 828 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0576023 s, 910 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040588 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0358297 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0420832 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0578324 s, 907 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0629199 s, 833 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0675248 s, 776 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0716113 s, 732 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0715101 s, 733 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.107522 s, 488 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10235 s, 512 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0435753 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272346 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345575 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339668 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0522082 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266697 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176333 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0508185 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407835 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0406099 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307022 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341755 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0434585 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333192 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257678 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0422546 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0390376 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275623 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0509021 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0378267 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0814721 s, 644 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347295 s, 1.5 GB/s 52428800 bytes (52 MB) copied, 0.0672469 s, 780 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0676403 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374999 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289403 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269073 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187098 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174283 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336476 s, 1.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521349 s, 1.0 GB/s 52428800 bytes (52 MB) copied, 0.0463629 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423505 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247663 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027426 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0455321 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342955 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350435 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466608 s, 1.1 GB/s 52428800 bytes (52 MB) copied, 0.0624335 s, 840 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391121 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341928 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0601304 s, 872 MB/s , 0.0413707 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338791 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328932 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0781345 s, 671 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244006 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226523 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368521 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345343 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408126 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031312 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034604 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253774 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015481 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0423189 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202988 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244745 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0362466 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320936 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0412877 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334109 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017439 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337846 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284426 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263462 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229073 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157524 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02609 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169905 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255963 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171207 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350953 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307859 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0374641 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292188 s, 1.8 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0248649 s, 2.1 GB/s , 0.0168539 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210588 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230799 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430349 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346072 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0356254 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032254 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351371 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0431915 s, 1.2 GB/s 52428800 bytes (52 MB) copied, 0.0335348 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438662 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030963 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.045439 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289908 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189512 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184111 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215966 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451412 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655878 s, 799 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0436985 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.086341 s, 607 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431078 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379693 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0610366 s, 859 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0962388 s, 545 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257549 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.070909 s, 739 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0694519 s, 755 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.108086 s, 485 MB/s 52428800 bytes (52 MB) copied, 0.0765517 s, 685 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0655645 s, 800 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.061689 s, 850 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481633 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209662 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.158013 s, 332 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348614 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0596431 s, 879 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0874951 s, 599 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0979898 s, 535 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0562041 s, 933 MB/s 52428800 bytes (52 MB) copied, 0.0405274 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.109886 s, 477 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198608 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0393888 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0769593 s, 681 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341537 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340268 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0449916 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271616 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0924054 s, 567 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280951 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306818 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0373441 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0724699 s, 723 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289513 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.0270222 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0177437 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0352042 s, 1.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184348 s, 2.8 GB/s 52428800 bytes (52 MB) copied, 0.0396326 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290927 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249574 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0598694 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0554775 s, 945 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0496327 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028112 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354025 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203566 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0707509 s, 741 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0501959 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.058858 s, 891 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330939 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175007 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027145 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211522 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299869 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236291 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186179 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238348 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202058 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017821 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.038799 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410109 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221446 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0457748 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224098 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213517 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019653 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353065 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167785 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0400631 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173402 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257109 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0353473 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157133 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332493 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376984 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480857 s, 1.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0889438 s, 589 MB/s 52428800 bytes (52 MB) copied, 0.101147 s, 518 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0427784 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0569191 s, 921 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.10192 s, 514 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.112409 s, 466 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132954 s, 394 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0969152 s, 541 MB/s 52428800 bytes (52 MB) copied, 0.100798 s, 520 MB/s 52428800 bytes (52 MB) copied, 0.10455 s, 501 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0678579 s, 773 MB/s 52428800 bytes (52 MB) copied, 0.0624883 s, 839 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116076 s, 452 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0682117 s, 769 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11446 s, 458 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160831 s, 326 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120982 s, 433 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.067105 s, 781 MB/s 52428800 bytes (52 MB) copied, 0.0676303 s, 775 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0863931 s, 607 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553861 s, 947 MB/s 52428800 bytes (52 MB) copied, 0.0381508 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332774 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157209 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0557972 s, 940 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0466964 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248643 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250992 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0382512 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0585779 s, 895 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0865175 s, 606 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985814 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158243 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0736526 s, 712 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.078632 s, 667 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336792 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037631 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132108 s, 397 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0491871 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0553527 s, 947 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.059032 s, 888 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0804168 s, 652 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0672802 s, 779 MB/s 52428800 bytes (52 MB) copied, 0.083393 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054463 s, 963 MB/s 52428800 bytes (52 MB) copied, 0.143563 s, 365 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142887 s, 367 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.167759 s, 313 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.161851 s, 324 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.18167 s, 289 MB/s 52428800 bytes (52 MB) copied50+0 records in 50+0 records out , 0.118106 s, 444 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130173 s, 403 MB/s 52428800 bytes (52 MB) copied, 0.179965 s, 291 MB/s 52428800 bytes (52 MB) copied, 0.154901 s, 338 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430459 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.14124 s, 371 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11887 s, 441 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.150987 s, 347 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.118242 s, 443 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135449 s, 387 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278966 s, 1.9 GB/s , 0.134653 s, 389 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138267 s, 379 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.136068 s, 385 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.130397 s, 402 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0894749 s, 586 MB/s /mnt/lustre/f45.sanity-pcc composite_header: lcm_magic: 0x0BD60BD0 lcm_size: 256 lcm_flags: ro,pccro lcm_layout_gen: 2 lcm_mirror_count: 2 lcm_entry_count: 2 components: - lcme_id: 65537 lcme_mirror_id: 1 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 128 lcme_size: 56 sub_layout: lmm_magic: 0x0BD10BD0 lmm_seq: 0x2000059f1 lmm_object_id: 0x8 lmm_fid: [0x2000059f1:0x8:0x0] lmm_stripe_count: 1 lmm_stripe_size: 4194304 lmm_pattern: raid0 lmm_layout_gen: 0 lmm_stripe_offset: 1 lmm_objects: - 0: { l_ost_idx: 1, l_fid: [0x280000400:0x1a5:0x0] } - lcme_id: 131074 lcme_mirror_id: 2 lcme_flags: init lcme_extent.e_start: 0 lcme_extent.e_end: EOF lcme_offset: 184 lcme_size: 72 sub_layout: lhm_magic: 0x0BD70BD0 lhm_pattern: hsm lhm_length: 56 lhm_type: 0x00000003 (pccro) lhm_flags: exists,archived,pccro,0x109 lhm_archive_id: 0 lhm_archive_ver: 0 lhm_archive_uuid: '' llite.lustre-ffff8800a8819800.stats= snapshot_time 1746143288.361386173 secs.nsecs start_time 1746143227.800742022 secs.nsecs elapsed_time 60.560644151 secs.nsecs read_bytes 1204 samples [bytes] 20480 1048576 1258291200 1318845876797440 read 152150 samples [usecs] 128 304881 117645542 1668596439392 ioctl 1 samples [reqs] open 3044 samples [usecs] 4 29253 322255 5845203065 close 3044 samples [usecs] 5 42852 369211 4658799597 seek 3042 samples [usecs] 2 757 16161 1401943 setattr 1 samples [usecs] 42428 42428 42428 1800135184 getattr 32 samples [usecs] 122 31320 92035 2120255675 mknod 1 samples [usecs] 14973 14973 14973 224190729 inode_permission 12197 samples [usecs] 1 13902 1114120 1620909866 opencount 3045 samples [reqs] 1 3044 4634491 9406469871 openclosetime 3043 samples [usecs] 2 137024 55750024 2109354813440 pcc_attach 1 samples [reqs] pcc_hit_bytes 3019 samples [bytes] 52428800 52428800 158282547200 8298564010639360000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 llite.lustre-ffff8800af4f4800.stats= snapshot_time 1746143288.361499126 secs.nsecs start_time 1746143227.800762632 secs.nsecs elapsed_time 60.560736494 secs.nsecs read_bytes 950 samples [bytes] 1048576 1048576 996147200 1044536046387200 read 151750 samples [usecs] 128 119013 118840782 1327601486776 open 3035 samples [usecs] 3 18185 98133 664097825 close 3035 samples [usecs] 6 37349 329671 3339848697 seek 3034 samples [usecs] 2 8494 25747 78245733 inode_permission 6068 samples [usecs] 1 70950 468317 5632179291 opencount 3035 samples [reqs] 1 3019 4558706 9176642986 openclosetime 3018 samples [usecs] 8 121500 53773821 1870339079295 pcc_attach 1 samples [reqs] pcc_hit_bytes 3016 samples [bytes] 52428800 52428800 158125260800 8290317673431040000 pcc_attach_bytes 1 samples [bytes] 52428800 52428800 52428800 2748779069440000 attach 2 detach 0 auto_attach 0 llite.lustre-ffff8800a8819800.pcc_async_threshold=1073741824 llite.lustre-ffff8800af4f4800.pcc_async_threshold=1073741824 /mnt/pcc.d45.sanity-pcc.2 is idle now /mnt/pcc.d45.sanity-pcc is idle now PASS 45 (66s) == sanity-pcc test 46: Verify PCC mode setting works correctly ========================================================== 19:48:11 (1746143291) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0826143 s, 762 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d46.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 0e306733-ca75-4af8-8f0b-3f242800366c Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f46.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=0e306733-ca75-4af8-8f0b-3f242800366c (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d46.sanity-pcc/d46.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} running as uid/gid/euid/egid 500/500/500/500, groups: 0 [id] uid=500(sanityusr) gid=500(sanityusr) groups=500(sanityusr),0(root) Mode: 0400 llite.lustre-ffff8800a8819800.pcc_mode=0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800a8819800.pcc_mode=0400 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] lfs pcc pcc: cannot attach the file to PCC with ID 0 failed: Operation not permitted (1) attach: cannot attach '/mnt/lustre/f46.sanity-pcc' to PCC with attach ID '0': Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ llite.lustre-ffff8800a8819800.pcc_mode=0004 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [stat] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 1 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0664/-rw-rw-r--) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-01 19:48:13.000000000 -0400 Modify: 2025-05-01 19:48:12.000000000 -0400 Change: 2025-05-01 19:48:13.000000000 -0400 Birth: - running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800a8819800.pcc_mode=0040 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [cat] [/mnt/lustre/f46.sanity-pcc] QQQQQ file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [attach] [-r] [/mnt/lustre/f46.sanity-pcc] File: '/mnt/lustre/f46.sanity-pcc' Size: 6 Blocks: 4 IO Block: 4194304 regular file Device: 2c54f966h/743766374d Inode: 144115574371254282 Links: 1 Access: (0660/-rw-rw----) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2025-05-01 19:48:13.000000000 -0400 Modify: 2025-05-01 19:48:12.000000000 -0400 Change: 2025-05-01 19:48:13.000000000 -0400 Birth: - file: /mnt/lustre/f46.sanity-pcc, type: readonly, PCC_file: /d46.sanity-pcc/59fb/0x2000059f1:0xa:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 0 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [detach] [/mnt/lustre/f46.sanity-pcc] llite.lustre-ffff8800a8819800.pcc_mode=0400 llite.lustre-ffff8800a8819800.pcc_mode=0400 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d46.sanity-pcc is idle now PASS 46 (4s) == sanity-pcc test 47: mtime should be kept once file attached into PCC ========================================================== 19:48:15 (1746143295) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0789463 s, 797 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d47.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: bec9cb70-1b69-4404-80d3-ddcd64cc32b1 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f47.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=bec9cb70-1b69-4404-80d3-ddcd64cc32b1 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d47.sanity-pcc/d47.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} QQQQQ /home/green/git/lustre-release/lustre/utils/lfs pcc state /mnt/lustre/f47.sanity-pcc | grep -E -c 'type: readonly' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d47.sanity-pcc is idle now PASS 47 (7s) == sanity-pcc test 48: PCC state should check whether the file in local PCC cache ========================================================== 19:48:22 (1746143302) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0864675 s, 728 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d48.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 575c2c43-b973-4ada-b4f2-68db6783f0f3 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f48.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=575c2c43-b973-4ada-b4f2-68db6783f0f3 (extents) (64bit) (large files) (huge files) QQQQQ file: /mnt/lustre/f48.sanity-pcc, type: readonly, PCC_file: /d48.sanity-pcc/59fd/0x2000059f1:0xc:0x0 (unlinked), open_count: 0, flags: 4 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d48.sanity-pcc is idle now PASS 48 (4s) == sanity-pcc test 96: Auto attach from multiple read process on a node ========================================================== 19:48:26 (1746143306) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.102365 s, 615 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d96.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 5dbbdd37-435b-4add-b320-c54bdabda864 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f96.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5dbbdd37-435b-4add-b320-c54bdabda864 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d96.sanity-pcc/d96.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8819800.pcc_async_threshold=1G llite.lustre-ffff8800af4f4800.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401111 s, 131 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.448307 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.449984 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.469068 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.464588 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.475726 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0483779 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206267 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334719 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286668 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240057 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274033 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280495 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152557 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222706 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153758 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193205 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368522 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196854 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222434 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249958 s, 2.1 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0273826 s, 1.9 GB/s 52428800 bytes (52 MB) copied, 0.0200472 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267478 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232484 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251856 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162423 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0337646 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332195 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425041 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0391902 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297258 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286252 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429365 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294519 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397138 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149919 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460813 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410365 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219286 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182639 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269851 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195929 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0439914 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0499218 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462758 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0379415 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274948 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281667 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326187 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288435 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263026 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258333 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288963 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222017 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197724 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311784 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020071 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179523 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226651 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260631 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257628 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310416 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193312 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236455 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251962 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192564 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196043 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175369 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181865 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230047 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029945 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250695 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225393 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201032 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316906 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248221 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169255 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193966 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022878 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322891 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246511 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354322 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202185 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256949 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161819 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234113 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202869 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235024 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022397 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.0205995 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351187 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191687 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208922 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183994 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388318 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271324 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258946 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249595 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231336 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300784 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233418 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192094 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188574 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311012 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347481 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017705 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235097 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301801 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260299 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174925 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0748496 s, 700 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438496 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0357147 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355506 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320346 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313877 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192364 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225313 s, 2.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.025612 s, 2.0 GB/s , 0.0190105 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407164 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.037813 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259416 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311116 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195296 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187702 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259127 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230034 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345529 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163984 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246268 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034885 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0527051 s, 995 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264361 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212127 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204415 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253249 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236844 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324889 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284176 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243955 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304414 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298537 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232036 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241527 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247295 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184526 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230254 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409599 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233363 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288243 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264126 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240448 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306418 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215928 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183525 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228624 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318082 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0383585 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233339 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028519 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0360718 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207674 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169006 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225761 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193986 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211045 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165213 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220549 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192836 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187717 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211063 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022945 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017165 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241782 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176865 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269877 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250348 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016687 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0333602 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214413 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228225 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194067 s, 2.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0215553 s, 2.4 GB/s , 0.0212505 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249832 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252745 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018805 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211916 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193861 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171959 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205071 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260546 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182886 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170203 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021671 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209284 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182356 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017359 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200564 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223591 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020659 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272887 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229331 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222519 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025974 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220085 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265797 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195996 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263775 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266171 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195119 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270779 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206886 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018689 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153979 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202783 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233766 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250404 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290812 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196658 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279238 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260242 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257306 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187737 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229487 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403062 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023366 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429612 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029794 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0371955 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267502 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193216 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016906 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218771 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022877 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020552 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237327 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185541 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361483 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239091 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205806 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261926 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175309 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215538 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262763 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241105 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178861 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263005 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349468 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258425 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184507 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213044 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175234 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194652 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163779 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174474 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178844 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197638 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0163033 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199211 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180151 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178775 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192736 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231289 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241496 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174037 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183239 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183015 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226538 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179614 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202833 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262751 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178158 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213994 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249421 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208374 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01654 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276153 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226036 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191386 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201998 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159959 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187579 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204155 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201936 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184434 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228549 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170438 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174253 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182048 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183736 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0152393 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213285 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192732 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259907 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175096 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214964 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193169 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198087 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221652 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177282 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202502 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023773 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194877 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235789 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284697 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216343 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241998 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173122 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155282 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192202 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183296 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206624 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217057 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304263 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208543 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164505 s, 3.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0226838 s, 2.3 GB/s , 0.0244763 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202295 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176384 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205253 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207259 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171916 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208902 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185242 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233825 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255245 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209605 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174958 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021757 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0519958 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.054464 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233557 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024992 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177812 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198212 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194546 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175405 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182544 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199675 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212325 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160658 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207185 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229872 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190479 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216585 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249989 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021251 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208879 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195395 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161636 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0238611 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199957 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194582 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221852 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169691 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260657 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015075 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016628 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202251 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016755 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177836 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204279 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240769 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189812 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208594 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180904 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207194 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028758 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359873 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186401 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185286 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204139 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243196 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233744 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179061 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187904 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198268 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188602 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181025 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203428 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026256 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0376213 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252519 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02204 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167052 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023163 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183669 s, 2.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299924 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0348865 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209913 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192147 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212772 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206657 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231154 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293645 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219307 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207276 s, 2.5 GB/s 52428800 bytes (52 MB) copied, 0.0306798 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256513 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176019 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217304 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247565 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248008 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230526 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156223 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210022 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221294 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022917 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234338 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259069 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286522 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257108 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221729 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156807 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172591 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217334 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027892 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172567 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293101 s, 1.8 GB/s 52428800 bytes (52 MB) copied, 0.0307898 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0398866 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183765 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207644 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251315 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289199 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222139 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253167 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283608 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217135 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197164 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284551 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0322943 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429805 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308359 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.05986 s, 876 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022803 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212114 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156399 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202271 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235523 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184417 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178686 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174423 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155576 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290706 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190252 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240371 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233099 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301237 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194056 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193768 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270378 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017702 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253336 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0336196 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247946 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294541 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025082 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247726 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186815 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04337 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153536 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317003 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0399848 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256325 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294356 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212079 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191085 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196181 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018665 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186294 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211789 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155685 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017197 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212425 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191217 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020397 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305575 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193604 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221129 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212023 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221046 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223341 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155673 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164174 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228419 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192585 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207534 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026641 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259706 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183504 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340061 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195615 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211914 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343673 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034946 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0280968 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223884 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268159 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276765 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219644 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217227 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0421983 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294026 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181077 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176766 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182312 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306058 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313324 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177413 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298921 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245559 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210751 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0431412 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154829 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255413 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199213 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0168607 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181944 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233406 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235928 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186564 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0258928 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0350559 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289881 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0419192 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257425 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01808 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365041 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0277344 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331881 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311687 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250712 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296561 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030719 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172017 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160639 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177695 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205768 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153202 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189599 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0153785 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437664 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310363 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031093 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377513 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190683 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213094 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184863 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0151459 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215254 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021124 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313448 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.04583 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402181 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228553 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225508 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241036 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0325083 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354914 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329616 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0381681 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368054 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305659 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328494 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195826 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338058 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283793 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246392 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234046 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159302 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167887 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254949 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164213 s, 3.2 GB/s , 0.0164048 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338663 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028389 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0462295 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236889 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0451955 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397062 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161813 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347516 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187219 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0309206 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199313 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326556 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185405 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188449 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179107 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020881 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198435 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016428 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040064 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032854 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242221 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278136 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369025 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244937 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210833 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194057 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197003 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259231 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0304224 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241839 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270245 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240131 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211466 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0160022 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214383 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167558 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174362 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0321845 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0410362 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0195394 s, 2.7 GB/s , 0.0345271 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195919 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247112 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025838 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156536 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015371 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235313 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180495 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172594 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227411 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192503 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193544 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219448 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177257 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245531 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233502 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021681 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161819 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297192 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202907 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265951 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264932 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0351294 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215523 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232837 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216658 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244365 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155438 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297071 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229717 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218151 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232389 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02643 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295063 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199973 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215755 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239825 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221849 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188145 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234644 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193312 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234855 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242184 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278432 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017065 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250658 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276398 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0349247 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0149944 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251164 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191004 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345797 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271568 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327216 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218552 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0484624 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015927 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298248 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236634 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268482 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266141 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244486 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204991 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179592 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265441 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285745 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181523 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215398 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227539 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263328 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186457 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178869 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263864 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204719 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185421 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215545 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203323 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.01579 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198158 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158058 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178255 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253394 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354465 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201855 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0219183 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256125 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233657 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199484 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167425 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203254 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242023 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269746 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296403 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162451 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180087 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156611 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222666 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018035 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167847 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183861 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230339 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233362 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248743 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021332 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021846 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269795 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186149 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242926 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.015983 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178773 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164553 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236586 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0262897 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228737 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0274076 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022512 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196755 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181977 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174062 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191242 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196055 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023342 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176319 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210481 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.042756 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194188 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031099 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217535 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193746 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237175 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022473 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039235 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345665 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241989 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186659 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211581 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202979 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177607 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184742 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223977 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179121 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0161287 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162678 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180639 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155012 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169509 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182028 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175687 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023207 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182468 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024302 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172674 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019777 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239097 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206553 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210578 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024698 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016298 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195118 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164336 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174218 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191615 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178747 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178823 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226624 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017863 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244772 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243447 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186869 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167544 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185554 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207669 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155886 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215491 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177878 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207692 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179974 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210669 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191806 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261921 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0334252 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234449 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213445 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174449 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018972 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204035 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237327 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020514 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199801 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218699 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287291 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271193 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210934 s, 2.5 GB/s , 0.0312329 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201326 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184599 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267415 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228236 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182537 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165352 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183297 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206312 s, 2.5 GB/s , 0.0227923 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201683 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02303 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192686 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261067 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182566 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176816 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0173342 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0148455 s, 3.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181338 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191935 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0171657 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0195193 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199304 s, 2.6 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.023609 s, 2.2 GB/s , 0.0261273 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021087 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0165424 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191252 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237376 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218086 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175727 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164914 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017654 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027355 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284173 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315603 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.034656 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0340838 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271004 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316775 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201306 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166625 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204401 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180504 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206503 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0338399 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0521056 s, 1.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181601 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181036 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017185 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0236786 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185467 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183295 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016223 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164399 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0234783 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341377 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0182406 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177283 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0320487 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215171 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0275859 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298919 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208334 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198566 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0253132 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210193 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183007 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017062 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016859 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225865 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249384 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194464 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205324 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246119 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027391 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259782 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198804 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295824 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191878 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206638 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272999 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0261485 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312807 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203672 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028151 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281129 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288153 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172547 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240454 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335941 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214155 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205732 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223851 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222315 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022239 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02361 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233796 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0159106 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361842 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199863 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02769 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187084 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266215 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189592 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192336 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0157306 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162116 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194519 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178364 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198155 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0289306 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227734 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017716 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169083 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193063 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240558 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256403 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215747 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186947 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278446 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225038 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.029514 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297277 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228572 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181668 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175899 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192767 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170453 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223476 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263859 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228419 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0295162 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199547 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0233452 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0215899 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186698 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0162867 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0178255 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166785 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312484 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388817 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315723 s, 1.7 GB/s 52428800 bytes (52 MB) copied, 0.0210243 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241686 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326987 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229224 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259829 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0306119 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0158768 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184814 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0207588 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0465927 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270194 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0229436 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242224 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0308603 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561504 s, 934 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248572 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0401351 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254174 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239754 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199469 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0189812 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169157 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214911 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0174371 s, 3.0 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0238292 s, 2.2 GB/s , 0.021183 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016612 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172497 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167942 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0167446 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260394 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.018366 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0243501 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0155103 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203123 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196831 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210953 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0244843 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199344 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166718 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199767 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259679 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216168 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223146 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237121 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220429 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0166107 s, 3.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175097 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0330645 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169554 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252521 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266768 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197399 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0248678 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209085 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023025 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192077 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227245 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228808 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205384 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264592 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0313755 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222211 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0179786 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021084 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202763 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0252753 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0188032 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228994 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0213623 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264441 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020137 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020009 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335902 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225786 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0287581 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0184522 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0263857 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239092 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0221268 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156705 s, 3.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332232 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199237 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255509 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276783 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0328493 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181562 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214852 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0286185 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176843 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027056 s, 1.9 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230105 s, 2.3 GB/s 52428800 bytes (52 MB) copied, 0.023457 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0181903 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208396 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247426 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0324187 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199041 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204744 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245688 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272445 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254296 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245647 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0305832 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297486 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0175147 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203457 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249379 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302619 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0240531 s, 2.2 GB/s , 0.0262416 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254949 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0201019 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209548 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205962 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208492 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031813 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276622 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0316426 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216145 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276891 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217542 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0187338 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0154925 s, 3.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209339 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326312 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0224565 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235704 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0198551 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0368219 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.016885 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.035921 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0183429 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0223677 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318272 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251704 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0194157 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028938 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327737 s, 1.6 GB/s , 0.0410838 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170749 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272859 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210442 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0301215 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0251172 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.020653 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032522 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0226681 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218317 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237026 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0169664 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0403051 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276983 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329253 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242689 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0370829 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0346597 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288909 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0264984 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0269094 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341218 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208057 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327859 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0317394 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0463535 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303393 s, 1.7 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0245614 s, 2.1 GB/s , 0.0407834 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239156 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172655 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202105 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265481 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0331893 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327186 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0217677 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.019861 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0170065 s, 3.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281429 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265444 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254198 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0658676 s, 796 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.030656 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256884 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.027225 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230043 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209223 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.031161 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172335 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186382 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.02265 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0177768 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0203386 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0180848 s, 2.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254205 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024174 s, 2.2 GB/s Finish ======== detach: must specify one or more FIDs Detach given files from the Persistent Client Cache. usage: lfs pcc detach [-k] FILE [...] lfs commands used to interact with PCC features: lfs pcc attach - attach given files to Persistent Client Cache lfs pcc attach_fid - attach given files into PCC by FID(s) lfs pcc state - display the PCC state for given files lfs pcc detach - detach given files from Persistent Client Cache lfs pcc detach_fid - detach given files from PCC by FID(s) lfs pcc delete - delete the PCC layout componenet for given files lfs pcc pin - pin give files for PCC lfs pcc unpin - unpin given files for PCC llite.lustre-ffff8800a8819800.pcc_async_threshold=1073741824 llite.lustre-ffff8800af4f4800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d96.sanity-pcc is idle now PASS 96 (65s) == sanity-pcc test 97: two mmap I/O and layout lock cancel ========================================================== 19:49:31 (1746143371) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0757977 s, 830 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d97.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 4e2cc2de-c884-43b2-9a2d-5c7d4d611f43 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f97.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=4e2cc2de-c884-43b2-9a2d-5c7d4d611f43 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d97.sanity-pcc/d97.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8819800.pcc_async_threshold=1G llite.lustre-ffff8800af4f4800.pcc_async_threshold=1G 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.50331 s, 104 MB/s T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... T2. mmap_cat /mnt/lustre/f97.sanity-pcc ... T1. mmap_cat /mnt/lustre/f97.sanity-pcc ... llite.lustre-ffff8800a8819800.pcc_async_threshold=1073741824 llite.lustre-ffff8800af4f4800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d97.sanity-pcc is idle now PASS 97 (124s) == sanity-pcc test 98: racer between auto attach and mmap I/O ========================================================== 19:51:35 (1746143495) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.084332 s, 746 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d98.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 298d0105-1b2d-41ff-bd2a-4abbadfc4b04 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f98.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=298d0105-1b2d-41ff-bd2a-4abbadfc4b04 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d98.sanity-pcc/d98.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} llite.lustre-ffff8800a8819800.pcc_async_threshold=0 llite.lustre-ffff8800af4f4800.pcc_async_threshold=0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.38651 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.46973 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.488817 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.498057 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0298575 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438013 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285326 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230742 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281607 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0284596 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250554 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192106 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293046 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.61805 s, 84.8 MB/s , 0.617956 s, 84.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.618192 s, 84.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0407108 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471753 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2583 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339993 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206129 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131796 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.131861 s, 398 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.111588 s, 470 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186897 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178005 s, 295 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.325945 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.274897 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.373709 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0377353 s, 1.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0718827 s, 729 MB/s , 0.0729337 s, 719 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0771358 s, 680 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.448035 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11914 s, 440 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.318427 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.265811 s, 197 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359695 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352787 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104264 s, 503 MB/s , 0.308405 s, 170 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120853 s, 434 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.341297 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.342381 s, 153 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.206298 s, 254 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.392619 s, 134 MB/s , 0.153848 s, 341 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258049 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.353834 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367665 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132491 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133183 s, 394 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.441504 s, 119 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.257336 s, 204 MB/s , 0.367602 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.370981 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.143927 s, 364 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0480724 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0778265 s, 674 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.274845 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.392638 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.2055 s, 255 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.284789 s, 184 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.205112 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.207298 s, 253 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.17808 s, 294 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.286215 s, 183 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.186839 s, 281 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0583626 s, 898 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.457993 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.46135 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.58686 s, 89.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16971 s, 309 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.177394 s, 296 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0561152 s, 934 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.482295 s, 109 MB/s , 0.388415 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.681788 s, 76.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.135058 s, 388 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.258408 s, 203 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437583 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.151683 s, 346 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.280201 s, 187 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.345372 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.424559 s, 123 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.255387 s, 205 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0402772 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.432968 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132448 s, 396 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132783 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367589 s, 143 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210203 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.032246 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0409894 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0249549 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0268823 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0307686 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0318078 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190537 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241054 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199855 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025092 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296619 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186663 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0208147 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242753 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205644 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.017741 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0225636 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0296562 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0281571 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310604 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0232276 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.153146 s, 342 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209129 s, 251 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.321562 s, 163 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0218605 s, 2.4 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.261847 s, 200 MB/s , 0.252287 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.336373 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0731173 s, 717 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.340453 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291676 s, 180 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292126 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292875 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.210529 s, 249 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.324103 s, 162 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162187 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290698 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.599044 s, 87.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.600083 s, 87.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.021972 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191579 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0270957 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256766 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241566 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220453 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0246863 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0327684 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279631 s, 1.9 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0172306 s, 3.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245475 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0190084 s, 2.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0425179 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0254153 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0497596 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0235241 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209436 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0339908 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023645 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323775 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206064 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0239432 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0397614 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0429168 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469629 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0341221 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0438093 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0230836 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0347508 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040032 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.593044 s, 88.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.34476 s, 152 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.37355 s, 140 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.116843 s, 449 MB/s , 0.312544 s, 168 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0199391 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985269 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.686679 s, 76.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.699544 s, 74.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.767835 s, 68.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.200814 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.314148 s, 167 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.330056 s, 159 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0212412 s, 2.5 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.117231 s, 447 MB/s , 0.0613173 s, 855 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.024179 s, 2.2 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.629901 s, 83.2 MB/s 52428800 bytes (52 MB) copied, 0.537617 s, 97.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.632012 s, 83.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0833762 s, 629 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0881119 s, 595 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0812855 s, 645 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348598 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.499677 s, 105 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.429419 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.353357 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.43436 s, 121 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.188507 s, 278 MB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.424078 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.214689 s, 244 MB/s 52428800 bytes (52 MB) copied, 0.284399 s, 184 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.614088 s, 85.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.371386 s, 141 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410242 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0544599 s, 963 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.16238 s, 323 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.203201 s, 258 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.449459 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239706 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.63386 s, 82.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.549296 s, 95.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339393 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0832844 s, 630 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.46815 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.736137 s, 71.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.70993 s, 73.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.71448 s, 73.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.260509 s, 201 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259524 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204524 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0411762 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0415943 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0302303 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303798 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0348616 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025044 s, 2.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0365218 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323129 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0540026 s, 971 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329354 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0211754 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.022755 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0297042 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388876 s, 1.3 GB/s 50+0 records in 50+0 records out 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied52428800 bytes (52 MB) copied, 0.0262299 s, 2.0 GB/s , 0.0268599 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.040242 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.052444 s, 1000 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0564299 s, 929 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0779149 s, 673 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0354441 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.036071 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202202 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0355273 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0476991 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197319 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0493523 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351364 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.483545 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.58714 s, 89.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.274513 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.274546 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.461786 s, 114 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.508828 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.733798 s, 71.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.751238 s, 69.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.105227 s, 498 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.269781 s, 194 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.357708 s, 147 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.578476 s, 90.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.67152 s, 78.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.331938 s, 158 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.21229 s, 247 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.132857 s, 395 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.142506 s, 368 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.749815 s, 69.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.565532 s, 92.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245608 s, 213 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0967471 s, 542 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.165519 s, 317 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.386505 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162978 s, 322 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.173957 s, 301 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.391072 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.404142 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.361215 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.528761 s, 99.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.181559 s, 289 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.183116 s, 286 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.346399 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0981537 s, 534 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0985388 s, 532 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.384073 s, 137 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.296625 s, 177 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.330522 s, 159 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.369237 s, 142 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.373564 s, 140 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0345717 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.26309 s, 199 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.361787 s, 145 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0209846 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.552194 s, 94.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.564315 s, 92.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.531708 s, 98.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401928 s, 130 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.298549 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.478244 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430146 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0319323 s, 1.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.354536 s, 148 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0923811 s, 568 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0652377 s, 804 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0454733 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.463061 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0227962 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.039626 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0202504 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0369764 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0240258 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0292465 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0290416 s, 1.8 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0214358 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231852 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0343984 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231344 s, 2.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216887 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0388055 s, 1.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0222808 s, 2.4 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210946 s, 2.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0342798 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0437272 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0486698 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0471339 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0200852 s, 2.6 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0242688 s, 2.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0359057 s, 1.5 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.386381 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.573002 s, 91.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.259019 s, 202 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.822807 s, 63.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.572377 s, 91.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.574573 s, 91.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.672571 s, 78.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.201157 s, 261 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.912191 s, 57.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.01785 s, 51.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.549876 s, 95.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.56105 s, 93.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.825466 s, 63.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.113613 s, 461 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.297573 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.533384 s, 98.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.651602 s, 80.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.924157 s, 56.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.931036 s, 56.3 MB/s llite.lustre-ffff8800a8819800.pcc_async_threshold=1073741824 llite.lustre-ffff8800af4f4800.pcc_async_threshold=1073741824 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d98.sanity-pcc is idle now PASS 98 (66s) == sanity-pcc test 99: race among unlink | mmap read | write | detach for PCC-RO file ========================================================== 19:52:41 (1746143561) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0932339 s, 675 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: a62434e4-dc1c-4772-a89b-987d4f8a60a4 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f99.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=a62434e4-dc1c-4772-a89b-987d4f8a60a4 (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477649 s, 110 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515506 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.657579 s, 79.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.612246 s, 85.6 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.667558 s, 78.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.532066 s, 98.5 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.0462 s, 50.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0231014 s, 2.3 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.522168 s, 100 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.464601 s, 113 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.595293 s, 88.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577417 s, 90.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.677879 s, 77.3 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.507983 s, 103 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.948012 s, 55.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260449 s, 2.0 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.89302 s, 58.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.03818 s, 50.5 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.764978 s, 68.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0185979 s, 2.8 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.538645 s, 97.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.681011 s, 77.0 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.659737 s, 79.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0294554 s, 1.8 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.650897 s, 80.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.840617 s, 62.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.640979 s, 81.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0617008 s, 850 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.292158 s, 179 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.656995 s, 79.8 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.723746 s, 72.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0300054 s, 1.7 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.967596 s, 54.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0260983 s, 2.0 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.483723 s, 108 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.648474 s, 80.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.655952 s, 79.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.57817 s, 90.7 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.697812 s, 75.1 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.483643 s, 108 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.712497 s, 73.6 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.367545 s, 143 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.475416 s, 110 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.599088 s, 87.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0283263 s, 1.9 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.297053 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.646129 s, 81.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.477729 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0272888 s, 1.9 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.406069 s, 129 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.59135 s, 88.7 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0241466 s, 2.2 GB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.340072 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.707385 s, 74.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.296973 s, 177 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.826469 s, 63.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.311539 s, 168 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0276764 s, 1.9 GB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355935 s, 147 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.637609 s, 82.2 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247906 s, 2.1 GB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.324745 s, 161 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.654499 s, 80.1 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.517514 s, 101 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.446698 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445879 s, 118 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.582575 s, 90.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196645 s, 2.7 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.778812 s, 67.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0164 s, 3.2 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.769442 s, 68.1 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.582356 s, 90.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577337 s, 90.8 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.026649 s, 2.0 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.70466 s, 74.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.349379 s, 150 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.59592 s, 88.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.697379 s, 75.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.08515 s, 48.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.063399 s, 827 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.601568 s, 87.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0329549 s, 1.6 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.612294 s, 85.6 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.684967 s, 76.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.682509 s, 76.8 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.588032 s, 89.2 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.162505 s, 323 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.277105 s, 189 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.673071 s, 77.9 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351457 s, 149 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.65354 s, 80.2 MB/s Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.724212 s, 72.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0186855 s, 2.8 GB/s Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.519586 s, 101 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0156954 s, 3.3 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.023489 s, 2.2 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.916556 s, 57.2 MB/s unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.382629 s, 137 MB/s Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.491644 s, 107 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219766 s, 239 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.578532 s, 90.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460143 s, 1.1 GB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Detach /mnt/lustre/f99.sanity-pcc ... Read /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.225221 s, 233 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.512342 s, 102 MB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.881326 s, 59.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0193266 s, 2.7 GB/s Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.484872 s, 108 MB/s Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.568103 s, 92.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.636389 s, 82.4 MB/s Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 Detach /mnt/lustre/f99.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0': No such file or directory Read /mnt/lustre/f99.sanity-pcc ... Detach /mnt/lustre/f99.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415108 s, 126 MB/s Detach /mnt/lustre/f99.sanity-pcc ... Unlink /mnt/pcc.d99.sanity-pcc/d99.sanity-pcc/59e2/0x2000059f1:0x13:0x0 ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.467585 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.413881 s, 127 MB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800a8819800.rpc_stats= snapshot_time: 1746143624.299527292 secs.nsecs start_time: 1746143094.514749730 secs.nsecs elapsed_time: 529.784777562 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1414 6 6 | 3 0 0 2: 11 0 6 | 0 0 0 4: 5 0 6 | 0 0 0 8: 19 0 7 | 0 0 0 16: 4609 22 29 | 0 0 0 32: 446 2 31 | 0 0 0 64: 638 3 34 | 0 0 0 128: 1507 7 42 | 0 0 0 256: 11783 57 100 | 3050 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 12436 60 60 | 341 11 11 2: 1896 9 70 | 899 29 40 3: 933 4 74 | 792 25 66 4: 756 3 78 | 444 14 81 5: 670 3 81 | 251 8 89 6: 535 2 84 | 157 5 94 7: 498 2 86 | 75 2 96 8: 493 2 89 | 45 1 98 9: 2114 10 99 | 47 1 99 10: 21 0 99 | 2 0 100 11: 9 0 99 | 0 0 100 12: 9 0 99 | 0 0 100 13: 10 0 99 | 0 0 100 14: 7 0 99 | 0 0 100 15: 6 0 99 | 0 0 100 16: 5 0 99 | 0 0 100 17: 4 0 99 | 0 0 100 18: 3 0 99 | 0 0 100 19: 3 0 99 | 0 0 100 20: 3 0 99 | 0 0 100 21: 3 0 99 | 0 0 100 22: 3 0 99 | 0 0 100 23: 2 0 99 | 0 0 100 24: 2 0 99 | 0 0 100 25: 2 0 99 | 0 0 100 26: 2 0 99 | 0 0 100 27: 2 0 99 | 0 0 100 28: 2 0 99 | 0 0 100 29: 2 0 99 | 0 0 100 30: 1 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 235 1 1 | 64 2 2 1: 86 0 1 | 0 0 2 2: 164 0 2 | 0 0 2 4: 18 0 2 | 0 0 2 8: 1 0 2 | 0 0 2 16: 156 0 3 | 0 0 2 32: 265 1 4 | 0 0 2 64: 474 2 6 | 0 0 2 128: 923 4 11 | 0 0 2 256: 476 2 13 | 61 1 4 512: 722 3 17 | 122 3 8 1024: 1598 7 25 | 244 7 16 2048: 3168 15 40 | 488 15 32 4096: 5933 29 69 | 976 31 64 8192: 6213 30 100 | 1098 35 100 osc.lustre-OST0000-osc-ffff8800af4f4800.rpc_stats= snapshot_time: 1746143624.299663708 secs.nsecs start_time: 1746143096.294544112 secs.nsecs elapsed_time: 528.005119596 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 3 6 6 | 0 0 0 2: 1 2 8 | 0 0 0 3: 1 2 10 | 0 0 0 4: 1 2 12 | 0 0 0 5: 1 2 14 | 0 0 0 6: 1 2 16 | 0 0 0 7: 1 2 18 | 0 0 0 8: 1 2 20 | 0 0 0 9: 40 80 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800a8819800.rpc_stats= snapshot_time: 1746143624.299729553 secs.nsecs start_time: 1746143094.516448145 secs.nsecs elapsed_time: 529.783281408 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 1 0 1 | 0 0 1 32: 0 0 1 | 0 0 1 64: 0 0 1 | 0 0 1 128: 0 0 1 | 0 0 1 256: 352 98 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 36 10 10 | 12 7 7 2: 25 7 17 | 76 49 57 3: 18 5 22 | 52 33 91 4: 18 5 27 | 11 7 98 5: 13 3 30 | 2 1 100 6: 15 4 35 | 0 0 100 7: 14 3 39 | 0 0 100 8: 21 5 44 | 0 0 100 9: 196 55 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 29 8 17 | 12 7 17 2048: 56 15 33 | 24 15 33 4096: 112 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff8800af4f4800.rpc_stats= snapshot_time: 1746143624.299795953 secs.nsecs start_time: 1746143096.294898399 secs.nsecs elapsed_time: 528.004897554 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 3 6 6 | 0 0 0 2: 1 2 8 | 0 0 0 3: 1 2 10 | 0 0 0 4: 1 2 12 | 0 0 0 5: 1 2 14 | 0 0 0 6: 1 2 16 | 0 0 0 7: 1 2 18 | 0 0 0 8: 2 4 22 | 0 0 0 9: 39 78 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99.sanity-pcc is idle now PASS 99 (65s) == sanity-pcc test 99b: race among unlink | two readers | write | detach for PCC-RO file ========================================================== 19:53:46 (1746143626) 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 0.267327 s, 784 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d99b.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/204800 done Creating filesystem with 204800 1k blocks and 51200 inodes Filesystem UUID: 5ab83f34-6b50-4e28-8691-8cddda94f94d Superblock backups stored on blocks: 8193, 24577, 40961, 57345, 73729 Allocating group tables: 0/25 done Writing inode tables: 0/25 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/25 done /tmp/f99b.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=5ab83f34-6b50-4e28-8691-8cddda94f94d (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.434748 s, 121 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.393831 s, 133 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.480328 s, 109 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.028226 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.428049 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0279422 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.445959 s, 118 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.348744 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.508493 s, 103 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.22459 s, 233 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.53735 s, 97.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245457 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.163736 s, 320 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.458516 s, 114 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.483712 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0311054 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.459254 s, 114 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339013 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.553344 s, 94.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0204395 s, 2.6 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.219363 s, 239 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.308458 s, 170 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.43718 s, 120 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.247013 s, 212 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0250922 s, 2.1 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.546247 s, 96.0 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.160772 s, 326 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303503 s, 1.7 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.435043 s, 121 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.509316 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.449054 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0220884 s, 2.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.451342 s, 116 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.3485 s, 150 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.358381 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0511017 s, 1.0 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.758549 s, 69.1 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.233054 s, 225 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0228072 s, 2.3 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.408771 s, 128 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.576595 s, 90.9 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0216245 s, 2.4 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359347 s, 146 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.359301 s, 146 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.736314 s, 71.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.166765 s, 314 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257699 s, 2.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.455138 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335917 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0974621 s, 538 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.5095 s, 103 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.389212 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.40505 s, 129 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265616 s, 2.0 GB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.565343 s, 92.7 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.323509 s, 162 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.110618 s, 474 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.646273 s, 81.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.335599 s, 156 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.168037 s, 312 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.644475 s, 81.4 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.341549 s, 154 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0335367 s, 1.6 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0481838 s, 1.1 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.762287 s, 68.8 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315168 s, 1.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0589702 s, 889 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.753787 s, 69.6 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.30199 s, 174 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.198981 s, 263 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.787689 s, 66.6 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.349846 s, 150 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0361629 s, 1.4 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.311823 s, 168 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.70548 s, 74.3 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0392703 s, 1.3 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.437171 s, 120 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.346736 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0430954 s, 1.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.743701 s, 70.5 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.274512 s, 191 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.120315 s, 436 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.612504 s, 85.6 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.521167 s, 101 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0326926 s, 1.6 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.546562 s, 95.9 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.317917 s, 165 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0310012 s, 1.7 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.808 s, 64.9 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.419966 s, 125 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285371 s, 1.8 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.583232 s, 89.9 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.530905 s, 98.8 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.504669 s, 104 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0267663 s, 2.0 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.531798 s, 98.6 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.448172 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133503 s, 393 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.592167 s, 88.5 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.291878 s, 180 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.303312 s, 173 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.463453 s, 113 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.228338 s, 230 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0196112 s, 2.7 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237522 s, 2.2 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.559539 s, 93.7 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.302937 s, 173 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.182796 s, 287 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.495558 s, 106 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.387705 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.423398 s, 124 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0176039 s, 3.0 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.461081 s, 114 MB/s unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.583477 s, 89.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191288 s, 2.7 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.468038 s, 112 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.283141 s, 185 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0266371 s, 2.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.6465 s, 81.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.58233 s, 90.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0293955 s, 1.8 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.5217 s, 100 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0256172 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.217946 s, 241 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.683884 s, 76.7 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.428805 s, 122 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.272258 s, 193 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.968771 s, 54.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.351499 s, 149 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.28766 s, 182 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.64213 s, 81.6 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.698737 s, 75.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.540984 s, 96.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0278662 s, 1.9 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.422948 s, 124 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.339264 s, 155 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.114674 s, 457 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.758162 s, 69.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.363042 s, 144 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.275402 s, 190 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.748231 s, 70.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.104977 s, 499 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.525338 s, 99.8 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.578489 s, 90.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206087 s, 2.5 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.194202 s, 270 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.380897 s, 138 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.638596 s, 82.1 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.13722 s, 382 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.025537 s, 2.1 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410396 s, 128 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.209062 s, 251 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0245292 s, 2.1 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0944108 s, 555 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.549095 s, 95.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255891 s, 2.0 GB/s Read /mnt/lustre/f99b.sanity-pcc ... Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.297784 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.370394 s, 142 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.692872 s, 75.7 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.138072 s, 380 MB/s Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.146759 s, 357 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.390968 s, 134 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0285448 s, 1.8 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.410419 s, 128 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.317659 s, 165 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.294375 s, 178 MB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.49904 s, 105 MB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.271271 s, 193 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.278481 s, 188 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.560723 s, 93.5 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... Read /mnt/lustre/f99b.sanity-pcc ... Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0460862 s, 1.1 GB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0315357 s, 1.7 GB/s Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.41318 s, 127 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.34836 s, 151 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0312061 s, 1.7 GB/s Unlink /mnt/pcc.d99b.sanity-pcc/d99b.sanity-pcc/59cc/0x2000059f1:0x3d:0x0 Read /mnt/lustre/f99b.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.485168 s, 108 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.11356 s, 462 MB/s Detach /mnt/lustre/f99b.sanity-pcc ... ==== DONE ==== 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0210343 s, 2.5 GB/s ==== DONE WIAT ==== osc.lustre-OST0000-osc-ffff8800a8819800.rpc_stats= snapshot_time: 1746143688.788562517 secs.nsecs start_time: 1746143094.514749730 secs.nsecs elapsed_time: 594.273812787 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 1414 5 5 | 3 0 0 2: 11 0 5 | 0 0 0 4: 5 0 5 | 0 0 0 8: 19 0 5 | 0 0 0 16: 4609 18 24 | 0 0 0 32: 446 1 26 | 0 0 0 64: 638 2 29 | 0 0 0 128: 1507 6 35 | 0 0 0 256: 15691 64 100 | 6100 99 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 13141 53 53 | 640 10 10 2: 2376 9 63 | 2067 33 44 3: 1293 5 69 | 1648 27 71 4: 1111 4 73 | 829 13 84 5: 986 4 77 | 431 7 92 6: 838 3 81 | 264 4 96 7: 767 3 84 | 113 1 98 8: 748 3 87 | 60 0 99 9: 2977 12 99 | 49 0 99 10: 23 0 99 | 2 0 100 11: 9 0 99 | 0 0 100 12: 9 0 99 | 0 0 100 13: 10 0 99 | 0 0 100 14: 7 0 99 | 0 0 100 15: 6 0 99 | 0 0 100 16: 5 0 99 | 0 0 100 17: 4 0 99 | 0 0 100 18: 3 0 99 | 0 0 100 19: 3 0 99 | 0 0 100 20: 3 0 99 | 0 0 100 21: 3 0 99 | 0 0 100 22: 3 0 99 | 0 0 100 23: 2 0 99 | 0 0 100 24: 2 0 99 | 0 0 100 25: 2 0 99 | 0 0 100 26: 2 0 99 | 0 0 100 27: 2 0 99 | 0 0 100 28: 2 0 99 | 0 0 100 29: 2 0 99 | 0 0 100 30: 1 0 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 337 1 1 | 125 2 2 1: 86 0 1 | 0 0 2 2: 164 0 2 | 0 0 2 4: 18 0 2 | 0 0 2 8: 1 0 2 | 0 0 2 16: 156 0 3 | 0 0 2 32: 265 1 4 | 0 0 2 64: 474 1 6 | 0 0 2 128: 923 3 9 | 0 0 2 256: 568 2 12 | 122 1 4 512: 905 3 16 | 244 3 8 1024: 1951 8 24 | 488 7 16 2048: 3799 15 39 | 976 15 32 4096: 7126 29 68 | 1952 31 64 8192: 7567 31 100 | 2196 35 100 osc.lustre-OST0000-osc-ffff8800af4f4800.rpc_stats= snapshot_time: 1746143688.788735122 secs.nsecs start_time: 1746143096.294544112 secs.nsecs elapsed_time: 592.494191010 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 3 6 6 | 0 0 0 2: 1 2 8 | 0 0 0 3: 1 2 10 | 0 0 0 4: 1 2 12 | 0 0 0 5: 1 2 14 | 0 0 0 6: 1 2 16 | 0 0 0 7: 1 2 18 | 0 0 0 8: 1 2 20 | 0 0 0 9: 40 80 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 osc.lustre-OST0001-osc-ffff8800a8819800.rpc_stats= snapshot_time: 1746143688.788803962 secs.nsecs start_time: 1746143094.516448145 secs.nsecs elapsed_time: 594.272355817 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 3 0 0 | 3 1 1 2: 0 0 0 | 0 0 1 4: 0 0 0 | 0 0 1 8: 0 0 0 | 0 0 1 16: 1 0 1 | 0 0 1 32: 0 0 1 | 0 0 1 64: 0 0 1 | 0 0 1 128: 0 0 1 | 0 0 1 256: 352 98 100 | 150 98 100 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 36 10 10 | 12 7 7 2: 25 7 17 | 76 49 57 3: 18 5 22 | 52 33 91 4: 18 5 27 | 11 7 98 5: 13 3 30 | 2 1 100 6: 15 4 35 | 0 0 100 7: 14 3 39 | 0 0 100 8: 21 5 44 | 0 0 100 9: 196 55 100 | 0 0 100 read write offset rpcs % cum % | rpcs % cum % 0: 12 3 3 | 6 3 3 1: 0 0 3 | 0 0 3 2: 0 0 3 | 0 0 3 4: 0 0 3 | 0 0 3 8: 0 0 3 | 0 0 3 16: 0 0 3 | 0 0 3 32: 0 0 3 | 0 0 3 64: 0 0 3 | 0 0 3 128: 0 0 3 | 0 0 3 256: 7 1 5 | 3 1 5 512: 14 3 9 | 6 3 9 1024: 29 8 17 | 12 7 17 2048: 56 15 33 | 24 15 33 4096: 112 31 64 | 48 31 64 8192: 126 35 100 | 54 35 100 osc.lustre-OST0001-osc-ffff8800af4f4800.rpc_stats= snapshot_time: 1746143688.788878789 secs.nsecs start_time: 1746143096.294898399 secs.nsecs elapsed_time: 592.493980390 secs.nsecs read RPCs in flight: 0 write RPCs in flight: 0 DIO RPCs in flight: 0 pending write pages: 0 pending read pages: 0 read write pages per rpc rpcs % cum % | rpcs % cum % 1: 0 0 0 | 0 0 0 2: 0 0 0 | 0 0 0 4: 0 0 0 | 0 0 0 8: 0 0 0 | 0 0 0 16: 0 0 0 | 0 0 0 32: 0 0 0 | 0 0 0 64: 0 0 0 | 0 0 0 128: 0 0 0 | 0 0 0 256: 50 100 100 | 0 0 0 read write rpcs in flight rpcs % cum % | rpcs % cum % 1: 3 6 6 | 0 0 0 2: 1 2 8 | 0 0 0 3: 1 2 10 | 0 0 0 4: 1 2 12 | 0 0 0 5: 1 2 14 | 0 0 0 6: 1 2 16 | 0 0 0 7: 1 2 18 | 0 0 0 8: 2 4 22 | 0 0 0 9: 39 78 100 | 0 0 0 read write offset rpcs % cum % | rpcs % cum % 0: 1 2 2 | 0 0 0 1: 0 0 2 | 0 0 0 2: 0 0 2 | 0 0 0 4: 0 0 2 | 0 0 0 8: 0 0 2 | 0 0 0 16: 0 0 2 | 0 0 0 32: 0 0 2 | 0 0 0 64: 0 0 2 | 0 0 0 128: 0 0 2 | 0 0 0 256: 1 2 4 | 0 0 0 512: 2 4 8 | 0 0 0 1024: 4 8 16 | 0 0 0 2048: 8 16 32 | 0 0 0 4096: 16 32 64 | 0 0 0 8192: 18 36 100 | 0 0 0 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d99b.sanity-pcc is idle now PASS 99b (65s) == sanity-pcc test 100: race among PCC unlink | read | write | detach for PCC-RO file ========================================================== 19:54:51 (1746143691) 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0817466 s, 770 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d100.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 72dfca9d-4bf9-4b22-ba75-dbf1f9717e8b Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f100.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=72dfca9d-4bf9-4b22-ba75-dbf1f9717e8b (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456771 s, 115 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.0500316 s, 62.9 MB/s 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0181074 s, 405 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401172 s, 131 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0197566 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0323217 s, 1.6 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.629958 s, 83.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.267281 s, 196 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0942051 s, 557 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0128749 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.422872 s, 124 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 35+0 records in 35+0 records out 36700160 bytes (37 MB) copied, 0.36328 s, 101 MB/s 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0137507 s, 534 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.609986 s, 86.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.426484 s, 123 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0196817 s, 533 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.515814 s, 102 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.266587 s, 197 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0205239 s, 2.6 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0257549 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0255082 s, 2.1 GB/s 21+0 records in 21+0 records out 22020096 bytes (22 MB) copied, 0.180658 s, 122 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.239778 s, 219 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.653406 s, 80.2 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.047512 s, 1.1 GB/s 29+0 records in 29+0 records out 30408704 bytes (30 MB) copied, 0.131511 s, 231 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.763917 s, 68.6 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.456801 s, 115 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.401682 s, 131 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.07713 s, 95.2 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.916937 s, 57.2 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.406416 s, 129 MB/s 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0151057 s, 486 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.552669 s, 94.9 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0546647 s, 19.2 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0112299 s, 93.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.58721 s, 89.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.116972 s, 448 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.415161 s, 126 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.108376 s, 261 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 23+0 records in 23+0 records out 24117248 bytes (24 MB) copied, 0.11201 s, 215 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.656399 s, 79.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.452356 s, 116 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.184996 s, 283 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00429639 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.409177 s, 128 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.221189 s, 237 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0952765 s, 550 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00285537 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00305931 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.464517 s, 113 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.248035 s, 211 MB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.01466 s, 286 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.541445 s, 96.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.355388 s, 148 MB/s 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.105966 s, 426 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.00205288 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 41+0 records in 41+0 records out 42991616 bytes (43 MB) copied, 0.167701 s, 256 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.503812 s, 104 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 17+0 records in 17+0 records out 17825792 bytes (18 MB) copied, 0.202749 s, 87.9 MB/s 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.0159671 s, 197 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.566509 s, 92.5 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.389662 s, 135 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.103799 s, 505 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 35+0 records in 35+0 records out 36700160 bytes (37 MB) copied, 0.0288304 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.547363 s, 95.8 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 28+0 records in 28+0 records out 29360128 bytes (29 MB) copied, 0.161156 s, 182 MB/s 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.0214279 s, 294 MB/s 33+0 records in 33+0 records out 34603008 bytes (35 MB) copied, 0.163698 s, 211 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.501904 s, 104 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0192518 s, 2.7 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0206675 s, 2.5 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0299206 s, 1.8 GB/s 11+0 records in 11+0 records out 11534336 bytes (12 MB) copied, 0.106805 s, 108 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 46+0 records in 46+0 records out 48234496 bytes (48 MB) copied, 0.177052 s, 272 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.546816 s, 95.9 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0271255 s, 1.9 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.662742 s, 79.1 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.312422 s, 168 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0247229 s, 2.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0288014 s, 1.8 GB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.19129 s, 236 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.490187 s, 107 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0397207 s, 26.4 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0111227 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.137186 s, 306 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.495183 s, 106 MB/s 24+0 records in 24+0 records out 25165824 bytes (25 MB) copied, 0.34505 s, 72.9 MB/s 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0177984 s, 295 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.542326 s, 96.7 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.178553 s, 294 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0195457 s, 0.0 kB/s 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0149054 s, 492 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.543045 s, 96.5 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 17+0 records in 17+0 records out 17825792 bytes (18 MB) copied, 0.277535 s, 64.2 MB/s 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0145278 s, 144 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.56028 s, 93.6 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.331476 s, 158 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 36+0 records in 36+0 records out 37748736 bytes (38 MB) copied, 0.148416 s, 254 MB/s 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.175065 s, 240 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.696207 s, 75.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.384666 s, 136 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.302531 s, 173 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 19+0 records in 19+0 records out 19922944 bytes (20 MB) copied, 0.113033 s, 176 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.57379 s, 91.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.326133 s, 161 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0481257 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.47633 s, 110 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0265742 s, 2.0 GB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0760819 s, 13.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... dd: warning: partial read (573440 bytes); suggest iflag=fullblock 0+2 records in 0+2 records out 1048576 bytes (1.0 MB) copied, 0.0139163 s, 75.3 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.560811 s, 93.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.176231 s, 298 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0237739 s, 2.2 GB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0197467 s, 53.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.625837 s, 83.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 27+0 records in 27+0 records out 28311552 bytes (28 MB) copied, 0.36621 s, 77.3 MB/s 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0209735 s, 250 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 43+0 records in 43+0 records out 45088768 bytes (45 MB) copied, 0.216837 s, 208 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.468237 s, 112 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 20+0 records in 20+0 records out 20971520 bytes (21 MB) copied, 0.391082 s, 53.6 MB/s 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.019224 s, 273 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.467895 s, 112 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0408067 s, 1.3 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0191562 s, 2.7 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 49+0 records in 49+0 records out 51380224 bytes (51 MB) copied, 0.298189 s, 172 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.594503 s, 88.2 MB/s 18+0 records in 18+0 records out 18874368 bytes (19 MB) copied, 0.359225 s, 52.5 MB/s 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0411251 s, 102 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.663764 s, 79.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.346994 s, 151 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0469982 s, 1.1 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 31+0 records in 31+0 records out 32505856 bytes (33 MB) copied, 0.375863 s, 86.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.80528 s, 65.1 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.504166 s, 104 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0426494 s, 1.2 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.521897 s, 100 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245988 s, 213 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.845725 s, 62.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.126124 s, 8.3 MB/s 35+0 records in 35+0 records out 36700160 bytes (37 MB) copied, 0.181403 s, 202 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.966263 s, 54.3 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.448657 s, 117 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.577416 s, 90.8 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 44+0 records in 44+0 records out 46137344 bytes (46 MB) copied, 0.163549 s, 282 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0251158 s, 41.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.352349 s, 149 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.245161 s, 214 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.797722 s, 65.7 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.175014 s, 300 MB/s 42+0 records in 42+0 records out 44040192 bytes (44 MB) copied, 0.230516 s, 191 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.582679 s, 90.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.961815 s, 54.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.35323 s, 148 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000407101 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0429311 s, 0.0 kB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.87815 s, 59.7 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.52867 s, 99.2 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0132718 s, 0.0 kB/s 40+0 records in 40+0 records out 41943040 bytes (42 MB) copied, 0.380652 s, 110 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0318248 s, 329 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.818805 s, 64.0 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 3+0 records in 3+0 records out 3145728 bytes (3.1 MB) copied, 0.083443 s, 37.7 MB/s 1+0 records in 1+0 records out 1048576 bytes (1.0 MB) copied, 0.0194258 s, 54.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.558348 s, 93.9 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.556808 s, 94.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.15406 s, 340 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.112068 s, 56.1 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 25+0 records in 25+0 records out 26214400 bytes (26 MB) copied, 0.0804885 s, 326 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.789574 s, 66.4 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.133917 s, 392 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.0104223 s, 604 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.609406 s, 86.0 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.35366 s, 148 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 7+0 records in 7+0 records out 7340032 bytes (7.3 MB) copied, 0.0243546 s, 301 MB/s unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.531753 s, 98.6 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 25+0 records in 25+0 records out 26214400 bytes (26 MB) copied, 0.54155 s, 48.4 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 5+0 records in 5+0 records out 5242880 bytes (5.2 MB) copied, 0.0211253 s, 248 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 1.11106 s, 47.2 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.873613 s, 60.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... 4+0 records in 4+0 records out 4194304 bytes (4.2 MB) copied, 0.0332629 s, 126 MB/s 6+0 records in 6+0 records out 6291456 bytes (6.3 MB) copied, 0.0567792 s, 111 MB/s Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 unlink: cannot unlink '/mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0': No such file or directory Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.639647 s, 82.0 MB/s 26+1 records in 26+1 records out 28221440 bytes (28 MB) copied, 0.336418 s, 83.9 MB/s 2+0 records in 2+0 records out 2097152 bytes (2.1 MB) copied, 0.0446328 s, 47.0 MB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 Detach /mnt/lustre/f100.sanity-pcc ... 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.787869 s, 66.5 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.440468 s, 119 MB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.000313285 s, 0.0 kB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.0764542 s, 0.0 kB/s Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 34+0 records in 34+0 records out 35651584 bytes (36 MB) copied, 0.0619948 s, 575 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.664495 s, 78.9 MB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0259741 s, 2.0 GB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0332725 s, 1.6 GB/s Detach /mnt/lustre/f100.sanity-pcc ... Detach /mnt/lustre/f100.sanity-pcc ... Unlink /mnt/pcc.d100.sanity-pcc/d100.sanity-pcc/59aa/0x2000059f1:0x5b:0x0 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0303642 s, 1.7 GB/s 0+0 records in 0+0 records out 0 bytes (0 B) copied, 0.138046 s, 0.0 kB/s 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.634195 s, 82.7 MB/s Cleanup PCC backend on /mnt/lustre /mnt/pcc.d100.sanity-pcc is idle now PASS 100 (65s) == sanity-pcc test 101a: Test auto attach in mount namespace (simulated container) ========================================================== 19:55:56 (1746143756) creating user namespace for 500 sanityu+ 21589 0.0 0.0 108060 364 pts/0 S+ 19:55 0:00 sleep 600 root 21591 0.0 0.0 114880 984 pts/0 S+ 19:55 0:00 grep sleep Created NS: child (sleep) pid=21589 in 0 seconds running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newuidmap] [21589] [0] [500] [1] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [newgidmap] [21589] [0] [500] [1] 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0809295 s, 648 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d101a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: 7b7bc319-03a8-4f65-b822-bb733dea1125 Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f101a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=7b7bc319-03a8-4f65-b822-bb733dea1125 (extents) (64bit) (large files) (huge files) Check for temp file in PCC mount Check for temp file in PCC mount from inside namespace Starting copytool 'agt1' on 'oleg413-client.virtnet' with cmdline 'lhsmtool_posix --archive-format=v2 --hsm-root=/mnt/pcc.d101a.sanity-pcc/d101a.sanity-pcc --daemon --pid-file=/var/run/lhsmtool_posix.pid --archive 2 "/mnt/lustre"' Verify open attach from inside mount namespace 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.0185406 s, 55.2 kB/s file: /mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc, type: readwrite, PCC_file: /d101a.sanity-pcc/598d/0x2000059f1:0x7c:0x0, open_count: 0, flags: 0 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af4f4800.lru_size=clear running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [multiop] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] [oc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [state] [/mnt/lustre/d101a.sanity-pcc/f101a.sanity-pcc] 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.000780771 s, 1.3 MB/s Verify auto attach at open from inside NS for RW-PCC ldlm.namespaces.lustre-MDT0000-mdc-ffff8800a8819800.lru_size=clear ldlm.namespaces.lustre-MDT0000-mdc-ffff8800af4f4800.lru_size=clear Waiting 200s for 'SUCCEED' Cleanup PCC backend on /mnt/lustre /mnt/pcc.d101a.sanity-pcc is idle now PASS 101a (9s) == sanity-pcc test 102: PCC-RO should not hange for io_uring I/O engine ========================================================== 19:56:05 (1746143765) SKIP: sanity-pcc test_102 Client OS does not support io_uring I/O engine SKIP 102 (1s) == sanity-pcc test 203: Verify attach/hit bytes statistics data ========================================================== 19:56:06 (1746143766) 10+0 records in 10+0 records out 10485760 bytes (10 MB) copied, 0.0165339 s, 634 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d203.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/10240 done Creating filesystem with 10240 1k blocks and 2560 inodes Filesystem UUID: e7aefd87-5e2f-427f-b382-433f53a3448e Superblock backups stored on blocks: 8193 Allocating group tables: 0/2 done Writing inode tables: 0/2 done Creating journal (2048 blocks): done Writing superblocks and filesystem accounting information: 0/2 done /tmp/f203.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=e7aefd87-5e2f-427f-b382-433f53a3448e (extents) (64bit) (large files) (huge files) pcc: - pccpath: /mnt/pcc.d203.sanity-pcc/d203.sanity-pcc hsmtool: posix_v2 rwid: 0 roid: 2 flags: ae autocache: projid={0} 1+0 records in 1+0 records out 1024 bytes (1.0 kB) copied, 0.00688873 s, 149 kB/s ==== stats ==== llite.lustre-ffff8800a8819800.stats= snapshot_time 1746143768.105582477 secs.nsecs start_time 1746143768.024367252 secs.nsecs elapsed_time 0.081215225 secs.nsecs read_bytes 1 samples [bytes] 1024 1024 1024 1048576 write_bytes 1 samples [bytes] 1024 1024 1024 1048576 read 3 samples [usecs] 150 3010 5317 13735249 write 1 samples [usecs] 3271 3271 3271 10699441 open 3 samples [usecs] 19 13519 13559 182764163 close 3 samples [usecs] 1763 3424 8427 25329545 getattr 2 samples [usecs] 58 4214 4272 17761160 mknod 1 samples [usecs] 14234 14234 14234 202606756 inode_permission 7 samples [usecs] 2 3690 3917 13633531 opencount 3 samples [reqs] 1 3 6 14 openclosetime 2 samples [usecs] 10531 15735 26266 358492186 pcc_attach 1 samples [reqs] pcc_hit_bytes 2 samples [bytes] 1024 1024 2048 2097152 pcc_attach_bytes 1 samples [bytes] 1024 1024 1024 1048576 llite.lustre-ffff8800af4f4800.stats= snapshot_time 1746143768.105697852 secs.nsecs start_time 1746143768.024388007 secs.nsecs elapsed_time 0.081309845 secs.nsecs attach_num: 1, attach_bytes: 1024, hit_num: 2, hit_bytes: 2048 Cleanup PCC backend on /mnt/lustre /mnt/pcc.d203.sanity-pcc is idle now PASS 203 (4s) == sanity-pcc test 204a: pin/unpin pcc flag ============== 19:56:11 (1746143771) 50+0 records in 50+0 records out 52428800 bytes (52 MB) copied, 0.0827794 s, 633 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204a.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/51200 done Creating filesystem with 51200 1k blocks and 12824 inodes Filesystem UUID: d769b7a8-b706-43a5-832e-9459af62b22c Superblock backups stored on blocks: 8193, 24577, 40961 Allocating group tables: 0/7 done Writing inode tables: 0/7 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/7 done /tmp/f204a.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=d769b7a8-b706-43a5-832e-9459af62b22c (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2,hsm: 100]" getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204a.sanity-pcc.dat trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204a.sanity-pcc is idle now PASS 204a (4s) == sanity-pcc test 204b: Permission check for the pin/unpin operation ========================================================== 19:56:15 (1746143775) mdt.lustre-MDT0000.enable_pin_gid=0 /home/green/git/lustre-release/lustre/tests/sanity-pcc.sh: illegal option -- p running as uid/gid/euid/egid 500/500/500/500, groups: 500 [touch] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc trusted.pin="[hsm: 100]" running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] unpin: cannot unpin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] pin: cannot pin '/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc' for PCC: Operation not permitted mdt.lustre-MDT0000.enable_pin_gid=-1 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=500 running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [pin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] running as uid/gid/euid/egid 500/500/500/500, groups: 500 [/home/green/git/lustre-release/lustre/utils/lfs] [pcc] [unpin] [-i] [100] [/mnt/lustre/d204b.sanity-pcc/f204b.sanity-pcc] mdt.lustre-MDT0000.enable_pin_gid=0 PASS 204b (6s) == sanity-pcc test 204c: PCC pin/unpin without attach ID specified ========================================================== 19:56:21 (1746143781) pin: failed to get id for '/mnt/lustre/f204c.sanity-pcc': No such file or directory 60+0 records in 60+0 records out 62914560 bytes (63 MB) copied, 0.0905428 s, 695 MB/s sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) umount: /mnt/pcc.d204c.sanity-pcc: not mounted sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,mode=755) securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755) tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb) configfs on /sys/kernel/config type configfs (rw,relatime) /dev/nbd0 on / type ext4 (ro,relatime,stripe=32,data=ordered) rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12965) hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) debugfs on /sys/kernel/debug type debugfs (rw,relatime) mqueue on /dev/mqueue type mqueue (rw,relatime) binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime) none on /mnt type ramfs (rw,relatime) none on /var/lib/stateless/writable type tmpfs (rw,relatime) /dev/vda on /home/green/git/lustre-release type squashfs (ro,relatime) none on /var/cache/man type tmpfs (rw,relatime) none on /var/log type tmpfs (rw,relatime) none on /var/lib/dbus type tmpfs (rw,relatime) none on /tmp type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) none on /var/tmp type tmpfs (rw,relatime) none on /var/lib/NetworkManager type tmpfs (rw,relatime) none on /var/lib/systemd/random-seed type tmpfs (rw,relatime) none on /var/spool type tmpfs (rw,relatime) none on /var/lib/nfs type tmpfs (rw,relatime) none on /var/lib/gssproxy type tmpfs (rw,relatime) none on /var/lib/logrotate type tmpfs (rw,relatime) none on /etc type tmpfs (rw,relatime) none on /var/lib/rsyslog type tmpfs (rw,relatime) none on /var/lib/dhclient type tmpfs (rw,relatime) 192.168.200.253:/exports/state/oleg413-client.virtnet on /var/lib/stateless/state type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/boot on /boot type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) 192.168.200.253:/exports/state/oleg413-client.virtnet/etc/kdump.conf on /etc/kdump.conf type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime) 192.168.200.253:/exports/testreports/51288/testresults/sanity-pcc-zfs-centos7_x86_64-centos7_x86_64 on /tmp/testlogs type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.204.13,local_lock=none,addr=192.168.200.253) tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=382032k,mode=700) /dev/vda on /usr/sbin/mount.lustre type squashfs (ro,relatime) 192.168.204.113@tcp:/lustre on /mnt/lustre type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) 192.168.204.113@tcp:/lustre on /mnt/lustre2 type lustre (rw,checksum,noencrypt,flock,lazystatfs,lruresize,nolock,statfs_project,nouser_fid2path,user_xattr,verbose) mke2fs 1.46.2.wc5 (26-Mar-2022) Suggestion: Use Linux kernel >= 3.18 for improved stability of the metadata and journal checksum features. Discarding device blocks: 1024/61440 done Creating filesystem with 61440 1k blocks and 15360 inodes Filesystem UUID: 321a6cc1-135c-468d-a365-14e2ea918cc2 Superblock backups stored on blocks: 8193, 24577, 40961, 57345 Allocating group tables: 0/8 done Writing inode tables: 0/8 done Creating journal (4096 blocks): done Writing superblocks and filesystem accounting information: 0/8 done /tmp/f204c.sanity-pcc: Linux rev 1.0 ext4 filesystem data, UUID=321a6cc1-135c-468d-a365-14e2ea918cc2 (extents) (64bit) (large files) (huge files) getfattr: Removing leading '/' from absolute path names # file: mnt/lustre/f204c.sanity-pcc trusted.pin="[hsm: 2]" getfattr: Removing leading '/' from absolute path names /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute /mnt/lustre/f204c.sanity-pcc: trusted.pin: No such attribute Cleanup PCC backend on /mnt/lustre /mnt/pcc.d204c.sanity-pcc is idle now PASS 204c (4s) == sanity-pcc test complete, duration 1821 sec =========== 19:56:25 (1746143785) === sanity-pcc: start cleanup 19:56:26 (1746143786) === Stopping clients: oleg413-client.virtnet /mnt/lustre2 (opts:) Stopping client oleg413-client.virtnet /mnt/lustre2 opts: === sanity-pcc: finish cleanup 19:56:29 (1746143789) ===